341 lines
12 KiB
C#
341 lines
12 KiB
C#
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
// SPDX-FileCopyrightText: 2025 StellaOps Contributors
|
|
|
|
namespace StellaOps.Parity.Tests.Storage;
|
|
|
|
/// <summary>
|
|
/// Detects drift in parity metrics when StellaOps falls behind competitors.
|
|
/// Triggers alerts when drift exceeds configured thresholds for a sustained period.
|
|
/// </summary>
|
|
public sealed class ParityDriftDetector
|
|
{
|
|
private readonly ParityResultStore _store;
|
|
private readonly DriftThresholds _thresholds;
|
|
private readonly int _requiredTrendDays;
|
|
|
|
public ParityDriftDetector(
|
|
ParityResultStore store,
|
|
DriftThresholds? thresholds = null,
|
|
int requiredTrendDays = 3)
|
|
{
|
|
_store = store ?? throw new ArgumentNullException(nameof(store));
|
|
_thresholds = thresholds ?? DriftThresholds.Default;
|
|
_requiredTrendDays = requiredTrendDays;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Analyzes recent results and returns any drift alerts.
|
|
/// </summary>
|
|
public async Task<DriftAnalysisResult> AnalyzeAsync(CancellationToken ct = default)
|
|
{
|
|
// Load last N days of results
|
|
var startTime = DateTime.UtcNow.AddDays(-_requiredTrendDays - 1);
|
|
var results = await _store.LoadResultsAsync(startTime: startTime, ct: ct);
|
|
|
|
if (results.Count == 0)
|
|
{
|
|
return new DriftAnalysisResult
|
|
{
|
|
AnalyzedAt = DateTime.UtcNow,
|
|
ResultCount = 0,
|
|
Alerts = [],
|
|
Summary = "No parity results available for analysis."
|
|
};
|
|
}
|
|
|
|
var alerts = new List<DriftAlert>();
|
|
|
|
// Analyze SBOM completeness drift
|
|
var sbomCompleteness = results
|
|
.Select(r => (r.Timestamp, r.SbomMetrics.PackageCompletenessRatio))
|
|
.ToList();
|
|
|
|
var sbomDrift = CalculateDrift(sbomCompleteness);
|
|
if (sbomDrift.HasDrift && sbomDrift.DriftAmount > _thresholds.SbomCompletenessThreshold)
|
|
{
|
|
alerts.Add(new DriftAlert
|
|
{
|
|
MetricName = "SBOM Package Completeness",
|
|
DriftType = DriftType.Declining,
|
|
BaselineValue = sbomDrift.BaselineValue,
|
|
CurrentValue = sbomDrift.CurrentValue,
|
|
DriftPercent = sbomDrift.DriftAmount * 100,
|
|
ThresholdPercent = _thresholds.SbomCompletenessThreshold * 100,
|
|
TrendDays = _requiredTrendDays,
|
|
Severity = GetAlertSeverity(sbomDrift.DriftAmount, _thresholds.SbomCompletenessThreshold),
|
|
Recommendation = "Review SBOM extraction logic; compare against Syft package detection."
|
|
});
|
|
}
|
|
|
|
// Analyze vulnerability recall drift
|
|
var vulnRecall = results
|
|
.Select(r => (r.Timestamp, r.VulnMetrics.Recall))
|
|
.ToList();
|
|
|
|
var recallDrift = CalculateDrift(vulnRecall);
|
|
if (recallDrift.HasDrift && recallDrift.DriftAmount > _thresholds.VulnRecallThreshold)
|
|
{
|
|
alerts.Add(new DriftAlert
|
|
{
|
|
MetricName = "Vulnerability Recall",
|
|
DriftType = DriftType.Declining,
|
|
BaselineValue = recallDrift.BaselineValue,
|
|
CurrentValue = recallDrift.CurrentValue,
|
|
DriftPercent = recallDrift.DriftAmount * 100,
|
|
ThresholdPercent = _thresholds.VulnRecallThreshold * 100,
|
|
TrendDays = _requiredTrendDays,
|
|
Severity = GetAlertSeverity(recallDrift.DriftAmount, _thresholds.VulnRecallThreshold),
|
|
Recommendation = "Check feed freshness; verify matcher logic for new CVE patterns."
|
|
});
|
|
}
|
|
|
|
// Analyze latency drift (StellaOps vs Grype P95)
|
|
var latencyRatio = results
|
|
.Select(r => (r.Timestamp, r.LatencyMetrics.StellaOpsVsGrypeRatio))
|
|
.ToList();
|
|
|
|
var latencyDrift = CalculateInverseDrift(latencyRatio); // Higher is worse for latency
|
|
if (latencyDrift.HasDrift && latencyDrift.DriftAmount > _thresholds.LatencyRatioThreshold)
|
|
{
|
|
alerts.Add(new DriftAlert
|
|
{
|
|
MetricName = "Latency vs Grype (P95)",
|
|
DriftType = DriftType.Increasing,
|
|
BaselineValue = latencyDrift.BaselineValue,
|
|
CurrentValue = latencyDrift.CurrentValue,
|
|
DriftPercent = latencyDrift.DriftAmount * 100,
|
|
ThresholdPercent = _thresholds.LatencyRatioThreshold * 100,
|
|
TrendDays = _requiredTrendDays,
|
|
Severity = GetAlertSeverity(latencyDrift.DriftAmount, _thresholds.LatencyRatioThreshold),
|
|
Recommendation = "Profile scanner hot paths; check for new allocations or I/O bottlenecks."
|
|
});
|
|
}
|
|
|
|
// Analyze PURL completeness drift
|
|
var purlCompleteness = results
|
|
.Select(r => (r.Timestamp, r.SbomMetrics.PurlCompletenessRatio))
|
|
.ToList();
|
|
|
|
var purlDrift = CalculateDrift(purlCompleteness);
|
|
if (purlDrift.HasDrift && purlDrift.DriftAmount > _thresholds.PurlCompletenessThreshold)
|
|
{
|
|
alerts.Add(new DriftAlert
|
|
{
|
|
MetricName = "PURL Completeness",
|
|
DriftType = DriftType.Declining,
|
|
BaselineValue = purlDrift.BaselineValue,
|
|
CurrentValue = purlDrift.CurrentValue,
|
|
DriftPercent = purlDrift.DriftAmount * 100,
|
|
ThresholdPercent = _thresholds.PurlCompletenessThreshold * 100,
|
|
TrendDays = _requiredTrendDays,
|
|
Severity = GetAlertSeverity(purlDrift.DriftAmount, _thresholds.PurlCompletenessThreshold),
|
|
Recommendation = "Verify PURL generation for all package types; check ecosystem-specific extractors."
|
|
});
|
|
}
|
|
|
|
// Analyze F1 score drift
|
|
var f1Score = results
|
|
.Select(r => (r.Timestamp, r.VulnMetrics.F1Score))
|
|
.ToList();
|
|
|
|
var f1Drift = CalculateDrift(f1Score);
|
|
if (f1Drift.HasDrift && f1Drift.DriftAmount > _thresholds.F1ScoreThreshold)
|
|
{
|
|
alerts.Add(new DriftAlert
|
|
{
|
|
MetricName = "Vulnerability F1 Score",
|
|
DriftType = DriftType.Declining,
|
|
BaselineValue = f1Drift.BaselineValue,
|
|
CurrentValue = f1Drift.CurrentValue,
|
|
DriftPercent = f1Drift.DriftAmount * 100,
|
|
ThresholdPercent = _thresholds.F1ScoreThreshold * 100,
|
|
TrendDays = _requiredTrendDays,
|
|
Severity = GetAlertSeverity(f1Drift.DriftAmount, _thresholds.F1ScoreThreshold),
|
|
Recommendation = "Balance precision/recall; check for false positive patterns."
|
|
});
|
|
}
|
|
|
|
var summary = alerts.Count switch
|
|
{
|
|
0 => $"No drift detected across {results.Count} results over {_requiredTrendDays}+ days.",
|
|
1 => $"1 drift alert detected. Investigate {alerts[0].MetricName}.",
|
|
_ => $"{alerts.Count} drift alerts detected. Review SBOM and vulnerability parity."
|
|
};
|
|
|
|
return new DriftAnalysisResult
|
|
{
|
|
AnalyzedAt = DateTime.UtcNow,
|
|
ResultCount = results.Count,
|
|
Alerts = alerts,
|
|
Summary = summary
|
|
};
|
|
}
|
|
|
|
private static DriftCalculation CalculateDrift(IReadOnlyList<(DateTime Timestamp, double Value)> series)
|
|
{
|
|
if (series.Count < 2)
|
|
{
|
|
return new DriftCalculation { HasDrift = false };
|
|
}
|
|
|
|
// Split into first half (baseline) and second half (current)
|
|
var midpoint = series.Count / 2;
|
|
var baseline = series.Take(midpoint).Select(x => x.Value).ToList();
|
|
var current = series.Skip(midpoint).Select(x => x.Value).ToList();
|
|
|
|
var baselineAvg = baseline.Average();
|
|
var currentAvg = current.Average();
|
|
|
|
// Drift is relative decline: (baseline - current) / baseline
|
|
if (baselineAvg <= 0)
|
|
{
|
|
return new DriftCalculation { HasDrift = false };
|
|
}
|
|
|
|
var drift = (baselineAvg - currentAvg) / baselineAvg;
|
|
|
|
return new DriftCalculation
|
|
{
|
|
HasDrift = drift > 0, // Only flag declining metrics
|
|
BaselineValue = baselineAvg,
|
|
CurrentValue = currentAvg,
|
|
DriftAmount = Math.Max(0, drift)
|
|
};
|
|
}
|
|
|
|
private static DriftCalculation CalculateInverseDrift(IReadOnlyList<(DateTime Timestamp, double Value)> series)
|
|
{
|
|
if (series.Count < 2)
|
|
{
|
|
return new DriftCalculation { HasDrift = false };
|
|
}
|
|
|
|
// Split into first half (baseline) and second half (current)
|
|
var midpoint = series.Count / 2;
|
|
var baseline = series.Take(midpoint).Select(x => x.Value).ToList();
|
|
var current = series.Skip(midpoint).Select(x => x.Value).ToList();
|
|
|
|
var baselineAvg = baseline.Average();
|
|
var currentAvg = current.Average();
|
|
|
|
// Drift is relative increase: (current - baseline) / baseline
|
|
if (baselineAvg <= 0)
|
|
{
|
|
return new DriftCalculation { HasDrift = false };
|
|
}
|
|
|
|
var drift = (currentAvg - baselineAvg) / baselineAvg;
|
|
|
|
return new DriftCalculation
|
|
{
|
|
HasDrift = drift > 0, // Only flag increasing metrics (bad for latency)
|
|
BaselineValue = baselineAvg,
|
|
CurrentValue = currentAvg,
|
|
DriftAmount = Math.Max(0, drift)
|
|
};
|
|
}
|
|
|
|
private static AlertSeverity GetAlertSeverity(double drift, double threshold)
|
|
{
|
|
var ratio = drift / threshold;
|
|
|
|
return ratio switch
|
|
{
|
|
>= 3.0 => AlertSeverity.Critical,
|
|
>= 2.0 => AlertSeverity.High,
|
|
>= 1.5 => AlertSeverity.Medium,
|
|
_ => AlertSeverity.Low
|
|
};
|
|
}
|
|
|
|
private sealed record DriftCalculation
|
|
{
|
|
public bool HasDrift { get; init; }
|
|
public double BaselineValue { get; init; }
|
|
public double CurrentValue { get; init; }
|
|
public double DriftAmount { get; init; }
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Thresholds for triggering drift alerts.
|
|
/// Values are relative (e.g., 0.05 = 5% drift).
|
|
/// </summary>
|
|
public sealed record DriftThresholds
|
|
{
|
|
/// <summary>
|
|
/// Default thresholds: 5% drift on key metrics.
|
|
/// </summary>
|
|
public static DriftThresholds Default => new()
|
|
{
|
|
SbomCompletenessThreshold = 0.05,
|
|
PurlCompletenessThreshold = 0.05,
|
|
VulnRecallThreshold = 0.05,
|
|
F1ScoreThreshold = 0.05,
|
|
LatencyRatioThreshold = 0.10 // 10% for latency
|
|
};
|
|
|
|
/// <summary>
|
|
/// SBOM package completeness drift threshold (default: 5%).
|
|
/// </summary>
|
|
public double SbomCompletenessThreshold { get; init; }
|
|
|
|
/// <summary>
|
|
/// PURL completeness drift threshold (default: 5%).
|
|
/// </summary>
|
|
public double PurlCompletenessThreshold { get; init; }
|
|
|
|
/// <summary>
|
|
/// Vulnerability recall drift threshold (default: 5%).
|
|
/// </summary>
|
|
public double VulnRecallThreshold { get; init; }
|
|
|
|
/// <summary>
|
|
/// Vulnerability F1 score drift threshold (default: 5%).
|
|
/// </summary>
|
|
public double F1ScoreThreshold { get; init; }
|
|
|
|
/// <summary>
|
|
/// Latency ratio drift threshold (default: 10%).
|
|
/// </summary>
|
|
public double LatencyRatioThreshold { get; init; }
|
|
}
|
|
|
|
public sealed record DriftAnalysisResult
|
|
{
|
|
public required DateTime AnalyzedAt { get; init; }
|
|
public required int ResultCount { get; init; }
|
|
public required IReadOnlyList<DriftAlert> Alerts { get; init; }
|
|
public required string Summary { get; init; }
|
|
|
|
public bool HasAlerts => Alerts.Count > 0;
|
|
public bool HasCriticalAlerts => Alerts.Any(a => a.Severity == AlertSeverity.Critical);
|
|
}
|
|
|
|
public sealed record DriftAlert
|
|
{
|
|
public required string MetricName { get; init; }
|
|
public required DriftType DriftType { get; init; }
|
|
public required double BaselineValue { get; init; }
|
|
public required double CurrentValue { get; init; }
|
|
public required double DriftPercent { get; init; }
|
|
public required double ThresholdPercent { get; init; }
|
|
public required int TrendDays { get; init; }
|
|
public required AlertSeverity Severity { get; init; }
|
|
public required string Recommendation { get; init; }
|
|
}
|
|
|
|
public enum DriftType
|
|
{
|
|
Declining,
|
|
Increasing
|
|
}
|
|
|
|
public enum AlertSeverity
|
|
{
|
|
Low,
|
|
Medium,
|
|
High,
|
|
Critical
|
|
}
|