// SPDX-License-Identifier: AGPL-3.0-or-later // SPDX-FileCopyrightText: 2025 StellaOps Contributors namespace StellaOps.Parity.Tests.Storage; /// /// Detects drift in parity metrics when StellaOps falls behind competitors. /// Triggers alerts when drift exceeds configured thresholds for a sustained period. /// public sealed class ParityDriftDetector { private readonly ParityResultStore _store; private readonly DriftThresholds _thresholds; private readonly int _requiredTrendDays; public ParityDriftDetector( ParityResultStore store, DriftThresholds? thresholds = null, int requiredTrendDays = 3) { _store = store ?? throw new ArgumentNullException(nameof(store)); _thresholds = thresholds ?? DriftThresholds.Default; _requiredTrendDays = requiredTrendDays; } /// /// Analyzes recent results and returns any drift alerts. /// public async Task AnalyzeAsync(CancellationToken ct = default) { // Load last N days of results var startTime = DateTime.UtcNow.AddDays(-_requiredTrendDays - 1); var results = await _store.LoadResultsAsync(startTime: startTime, ct: ct); if (results.Count == 0) { return new DriftAnalysisResult { AnalyzedAt = DateTime.UtcNow, ResultCount = 0, Alerts = [], Summary = "No parity results available for analysis." }; } var alerts = new List(); // Analyze SBOM completeness drift var sbomCompleteness = results .Select(r => (r.Timestamp, r.SbomMetrics.PackageCompletenessRatio)) .ToList(); var sbomDrift = CalculateDrift(sbomCompleteness); if (sbomDrift.HasDrift && sbomDrift.DriftAmount > _thresholds.SbomCompletenessThreshold) { alerts.Add(new DriftAlert { MetricName = "SBOM Package Completeness", DriftType = DriftType.Declining, BaselineValue = sbomDrift.BaselineValue, CurrentValue = sbomDrift.CurrentValue, DriftPercent = sbomDrift.DriftAmount * 100, ThresholdPercent = _thresholds.SbomCompletenessThreshold * 100, TrendDays = _requiredTrendDays, Severity = GetAlertSeverity(sbomDrift.DriftAmount, _thresholds.SbomCompletenessThreshold), Recommendation = "Review SBOM extraction logic; compare against Syft package detection." }); } // Analyze vulnerability recall drift var vulnRecall = results .Select(r => (r.Timestamp, r.VulnMetrics.Recall)) .ToList(); var recallDrift = CalculateDrift(vulnRecall); if (recallDrift.HasDrift && recallDrift.DriftAmount > _thresholds.VulnRecallThreshold) { alerts.Add(new DriftAlert { MetricName = "Vulnerability Recall", DriftType = DriftType.Declining, BaselineValue = recallDrift.BaselineValue, CurrentValue = recallDrift.CurrentValue, DriftPercent = recallDrift.DriftAmount * 100, ThresholdPercent = _thresholds.VulnRecallThreshold * 100, TrendDays = _requiredTrendDays, Severity = GetAlertSeverity(recallDrift.DriftAmount, _thresholds.VulnRecallThreshold), Recommendation = "Check feed freshness; verify matcher logic for new CVE patterns." }); } // Analyze latency drift (StellaOps vs Grype P95) var latencyRatio = results .Select(r => (r.Timestamp, r.LatencyMetrics.StellaOpsVsGrypeRatio)) .ToList(); var latencyDrift = CalculateInverseDrift(latencyRatio); // Higher is worse for latency if (latencyDrift.HasDrift && latencyDrift.DriftAmount > _thresholds.LatencyRatioThreshold) { alerts.Add(new DriftAlert { MetricName = "Latency vs Grype (P95)", DriftType = DriftType.Increasing, BaselineValue = latencyDrift.BaselineValue, CurrentValue = latencyDrift.CurrentValue, DriftPercent = latencyDrift.DriftAmount * 100, ThresholdPercent = _thresholds.LatencyRatioThreshold * 100, TrendDays = _requiredTrendDays, Severity = GetAlertSeverity(latencyDrift.DriftAmount, _thresholds.LatencyRatioThreshold), Recommendation = "Profile scanner hot paths; check for new allocations or I/O bottlenecks." }); } // Analyze PURL completeness drift var purlCompleteness = results .Select(r => (r.Timestamp, r.SbomMetrics.PurlCompletenessRatio)) .ToList(); var purlDrift = CalculateDrift(purlCompleteness); if (purlDrift.HasDrift && purlDrift.DriftAmount > _thresholds.PurlCompletenessThreshold) { alerts.Add(new DriftAlert { MetricName = "PURL Completeness", DriftType = DriftType.Declining, BaselineValue = purlDrift.BaselineValue, CurrentValue = purlDrift.CurrentValue, DriftPercent = purlDrift.DriftAmount * 100, ThresholdPercent = _thresholds.PurlCompletenessThreshold * 100, TrendDays = _requiredTrendDays, Severity = GetAlertSeverity(purlDrift.DriftAmount, _thresholds.PurlCompletenessThreshold), Recommendation = "Verify PURL generation for all package types; check ecosystem-specific extractors." }); } // Analyze F1 score drift var f1Score = results .Select(r => (r.Timestamp, r.VulnMetrics.F1Score)) .ToList(); var f1Drift = CalculateDrift(f1Score); if (f1Drift.HasDrift && f1Drift.DriftAmount > _thresholds.F1ScoreThreshold) { alerts.Add(new DriftAlert { MetricName = "Vulnerability F1 Score", DriftType = DriftType.Declining, BaselineValue = f1Drift.BaselineValue, CurrentValue = f1Drift.CurrentValue, DriftPercent = f1Drift.DriftAmount * 100, ThresholdPercent = _thresholds.F1ScoreThreshold * 100, TrendDays = _requiredTrendDays, Severity = GetAlertSeverity(f1Drift.DriftAmount, _thresholds.F1ScoreThreshold), Recommendation = "Balance precision/recall; check for false positive patterns." }); } var summary = alerts.Count switch { 0 => $"No drift detected across {results.Count} results over {_requiredTrendDays}+ days.", 1 => $"1 drift alert detected. Investigate {alerts[0].MetricName}.", _ => $"{alerts.Count} drift alerts detected. Review SBOM and vulnerability parity." }; return new DriftAnalysisResult { AnalyzedAt = DateTime.UtcNow, ResultCount = results.Count, Alerts = alerts, Summary = summary }; } private static DriftCalculation CalculateDrift(IReadOnlyList<(DateTime Timestamp, double Value)> series) { if (series.Count < 2) { return new DriftCalculation { HasDrift = false }; } // Split into first half (baseline) and second half (current) var midpoint = series.Count / 2; var baseline = series.Take(midpoint).Select(x => x.Value).ToList(); var current = series.Skip(midpoint).Select(x => x.Value).ToList(); var baselineAvg = baseline.Average(); var currentAvg = current.Average(); // Drift is relative decline: (baseline - current) / baseline if (baselineAvg <= 0) { return new DriftCalculation { HasDrift = false }; } var drift = (baselineAvg - currentAvg) / baselineAvg; return new DriftCalculation { HasDrift = drift > 0, // Only flag declining metrics BaselineValue = baselineAvg, CurrentValue = currentAvg, DriftAmount = Math.Max(0, drift) }; } private static DriftCalculation CalculateInverseDrift(IReadOnlyList<(DateTime Timestamp, double Value)> series) { if (series.Count < 2) { return new DriftCalculation { HasDrift = false }; } // Split into first half (baseline) and second half (current) var midpoint = series.Count / 2; var baseline = series.Take(midpoint).Select(x => x.Value).ToList(); var current = series.Skip(midpoint).Select(x => x.Value).ToList(); var baselineAvg = baseline.Average(); var currentAvg = current.Average(); // Drift is relative increase: (current - baseline) / baseline if (baselineAvg <= 0) { return new DriftCalculation { HasDrift = false }; } var drift = (currentAvg - baselineAvg) / baselineAvg; return new DriftCalculation { HasDrift = drift > 0, // Only flag increasing metrics (bad for latency) BaselineValue = baselineAvg, CurrentValue = currentAvg, DriftAmount = Math.Max(0, drift) }; } private static AlertSeverity GetAlertSeverity(double drift, double threshold) { var ratio = drift / threshold; return ratio switch { >= 3.0 => AlertSeverity.Critical, >= 2.0 => AlertSeverity.High, >= 1.5 => AlertSeverity.Medium, _ => AlertSeverity.Low }; } private sealed record DriftCalculation { public bool HasDrift { get; init; } public double BaselineValue { get; init; } public double CurrentValue { get; init; } public double DriftAmount { get; init; } } } /// /// Thresholds for triggering drift alerts. /// Values are relative (e.g., 0.05 = 5% drift). /// public sealed record DriftThresholds { /// /// Default thresholds: 5% drift on key metrics. /// public static DriftThresholds Default => new() { SbomCompletenessThreshold = 0.05, PurlCompletenessThreshold = 0.05, VulnRecallThreshold = 0.05, F1ScoreThreshold = 0.05, LatencyRatioThreshold = 0.10 // 10% for latency }; /// /// SBOM package completeness drift threshold (default: 5%). /// public double SbomCompletenessThreshold { get; init; } /// /// PURL completeness drift threshold (default: 5%). /// public double PurlCompletenessThreshold { get; init; } /// /// Vulnerability recall drift threshold (default: 5%). /// public double VulnRecallThreshold { get; init; } /// /// Vulnerability F1 score drift threshold (default: 5%). /// public double F1ScoreThreshold { get; init; } /// /// Latency ratio drift threshold (default: 10%). /// public double LatencyRatioThreshold { get; init; } } public sealed record DriftAnalysisResult { public required DateTime AnalyzedAt { get; init; } public required int ResultCount { get; init; } public required IReadOnlyList Alerts { get; init; } public required string Summary { get; init; } public bool HasAlerts => Alerts.Count > 0; public bool HasCriticalAlerts => Alerts.Any(a => a.Severity == AlertSeverity.Critical); } public sealed record DriftAlert { public required string MetricName { get; init; } public required DriftType DriftType { get; init; } public required double BaselineValue { get; init; } public required double CurrentValue { get; init; } public required double DriftPercent { get; init; } public required double ThresholdPercent { get; init; } public required int TrendDays { get; init; } public required AlertSeverity Severity { get; init; } public required string Recommendation { get; init; } } public enum DriftType { Declining, Increasing } public enum AlertSeverity { Low, Medium, High, Critical }