5100* tests strengthtenen work
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,340 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
// SPDX-FileCopyrightText: 2025 StellaOps Contributors
|
||||
|
||||
namespace StellaOps.Parity.Tests.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Detects drift in parity metrics when StellaOps falls behind competitors.
|
||||
/// Triggers alerts when drift exceeds configured thresholds for a sustained period.
|
||||
/// </summary>
|
||||
public sealed class ParityDriftDetector
|
||||
{
|
||||
private readonly ParityResultStore _store;
|
||||
private readonly DriftThresholds _thresholds;
|
||||
private readonly int _requiredTrendDays;
|
||||
|
||||
public ParityDriftDetector(
|
||||
ParityResultStore store,
|
||||
DriftThresholds? thresholds = null,
|
||||
int requiredTrendDays = 3)
|
||||
{
|
||||
_store = store ?? throw new ArgumentNullException(nameof(store));
|
||||
_thresholds = thresholds ?? DriftThresholds.Default;
|
||||
_requiredTrendDays = requiredTrendDays;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Analyzes recent results and returns any drift alerts.
|
||||
/// </summary>
|
||||
public async Task<DriftAnalysisResult> AnalyzeAsync(CancellationToken ct = default)
|
||||
{
|
||||
// Load last N days of results
|
||||
var startTime = DateTime.UtcNow.AddDays(-_requiredTrendDays - 1);
|
||||
var results = await _store.LoadResultsAsync(startTime: startTime, ct: ct);
|
||||
|
||||
if (results.Count == 0)
|
||||
{
|
||||
return new DriftAnalysisResult
|
||||
{
|
||||
AnalyzedAt = DateTime.UtcNow,
|
||||
ResultCount = 0,
|
||||
Alerts = [],
|
||||
Summary = "No parity results available for analysis."
|
||||
};
|
||||
}
|
||||
|
||||
var alerts = new List<DriftAlert>();
|
||||
|
||||
// Analyze SBOM completeness drift
|
||||
var sbomCompleteness = results
|
||||
.Select(r => (r.Timestamp, r.SbomMetrics.PackageCompletenessRatio))
|
||||
.ToList();
|
||||
|
||||
var sbomDrift = CalculateDrift(sbomCompleteness);
|
||||
if (sbomDrift.HasDrift && sbomDrift.DriftAmount > _thresholds.SbomCompletenessThreshold)
|
||||
{
|
||||
alerts.Add(new DriftAlert
|
||||
{
|
||||
MetricName = "SBOM Package Completeness",
|
||||
DriftType = DriftType.Declining,
|
||||
BaselineValue = sbomDrift.BaselineValue,
|
||||
CurrentValue = sbomDrift.CurrentValue,
|
||||
DriftPercent = sbomDrift.DriftAmount * 100,
|
||||
ThresholdPercent = _thresholds.SbomCompletenessThreshold * 100,
|
||||
TrendDays = _requiredTrendDays,
|
||||
Severity = GetAlertSeverity(sbomDrift.DriftAmount, _thresholds.SbomCompletenessThreshold),
|
||||
Recommendation = "Review SBOM extraction logic; compare against Syft package detection."
|
||||
});
|
||||
}
|
||||
|
||||
// Analyze vulnerability recall drift
|
||||
var vulnRecall = results
|
||||
.Select(r => (r.Timestamp, r.VulnMetrics.Recall))
|
||||
.ToList();
|
||||
|
||||
var recallDrift = CalculateDrift(vulnRecall);
|
||||
if (recallDrift.HasDrift && recallDrift.DriftAmount > _thresholds.VulnRecallThreshold)
|
||||
{
|
||||
alerts.Add(new DriftAlert
|
||||
{
|
||||
MetricName = "Vulnerability Recall",
|
||||
DriftType = DriftType.Declining,
|
||||
BaselineValue = recallDrift.BaselineValue,
|
||||
CurrentValue = recallDrift.CurrentValue,
|
||||
DriftPercent = recallDrift.DriftAmount * 100,
|
||||
ThresholdPercent = _thresholds.VulnRecallThreshold * 100,
|
||||
TrendDays = _requiredTrendDays,
|
||||
Severity = GetAlertSeverity(recallDrift.DriftAmount, _thresholds.VulnRecallThreshold),
|
||||
Recommendation = "Check feed freshness; verify matcher logic for new CVE patterns."
|
||||
});
|
||||
}
|
||||
|
||||
// Analyze latency drift (StellaOps vs Grype P95)
|
||||
var latencyRatio = results
|
||||
.Select(r => (r.Timestamp, r.LatencyMetrics.StellaOpsVsGrypeRatio))
|
||||
.ToList();
|
||||
|
||||
var latencyDrift = CalculateInverseDrift(latencyRatio); // Higher is worse for latency
|
||||
if (latencyDrift.HasDrift && latencyDrift.DriftAmount > _thresholds.LatencyRatioThreshold)
|
||||
{
|
||||
alerts.Add(new DriftAlert
|
||||
{
|
||||
MetricName = "Latency vs Grype (P95)",
|
||||
DriftType = DriftType.Increasing,
|
||||
BaselineValue = latencyDrift.BaselineValue,
|
||||
CurrentValue = latencyDrift.CurrentValue,
|
||||
DriftPercent = latencyDrift.DriftAmount * 100,
|
||||
ThresholdPercent = _thresholds.LatencyRatioThreshold * 100,
|
||||
TrendDays = _requiredTrendDays,
|
||||
Severity = GetAlertSeverity(latencyDrift.DriftAmount, _thresholds.LatencyRatioThreshold),
|
||||
Recommendation = "Profile scanner hot paths; check for new allocations or I/O bottlenecks."
|
||||
});
|
||||
}
|
||||
|
||||
// Analyze PURL completeness drift
|
||||
var purlCompleteness = results
|
||||
.Select(r => (r.Timestamp, r.SbomMetrics.PurlCompletenessRatio))
|
||||
.ToList();
|
||||
|
||||
var purlDrift = CalculateDrift(purlCompleteness);
|
||||
if (purlDrift.HasDrift && purlDrift.DriftAmount > _thresholds.PurlCompletenessThreshold)
|
||||
{
|
||||
alerts.Add(new DriftAlert
|
||||
{
|
||||
MetricName = "PURL Completeness",
|
||||
DriftType = DriftType.Declining,
|
||||
BaselineValue = purlDrift.BaselineValue,
|
||||
CurrentValue = purlDrift.CurrentValue,
|
||||
DriftPercent = purlDrift.DriftAmount * 100,
|
||||
ThresholdPercent = _thresholds.PurlCompletenessThreshold * 100,
|
||||
TrendDays = _requiredTrendDays,
|
||||
Severity = GetAlertSeverity(purlDrift.DriftAmount, _thresholds.PurlCompletenessThreshold),
|
||||
Recommendation = "Verify PURL generation for all package types; check ecosystem-specific extractors."
|
||||
});
|
||||
}
|
||||
|
||||
// Analyze F1 score drift
|
||||
var f1Score = results
|
||||
.Select(r => (r.Timestamp, r.VulnMetrics.F1Score))
|
||||
.ToList();
|
||||
|
||||
var f1Drift = CalculateDrift(f1Score);
|
||||
if (f1Drift.HasDrift && f1Drift.DriftAmount > _thresholds.F1ScoreThreshold)
|
||||
{
|
||||
alerts.Add(new DriftAlert
|
||||
{
|
||||
MetricName = "Vulnerability F1 Score",
|
||||
DriftType = DriftType.Declining,
|
||||
BaselineValue = f1Drift.BaselineValue,
|
||||
CurrentValue = f1Drift.CurrentValue,
|
||||
DriftPercent = f1Drift.DriftAmount * 100,
|
||||
ThresholdPercent = _thresholds.F1ScoreThreshold * 100,
|
||||
TrendDays = _requiredTrendDays,
|
||||
Severity = GetAlertSeverity(f1Drift.DriftAmount, _thresholds.F1ScoreThreshold),
|
||||
Recommendation = "Balance precision/recall; check for false positive patterns."
|
||||
});
|
||||
}
|
||||
|
||||
var summary = alerts.Count switch
|
||||
{
|
||||
0 => $"No drift detected across {results.Count} results over {_requiredTrendDays}+ days.",
|
||||
1 => $"1 drift alert detected. Investigate {alerts[0].MetricName}.",
|
||||
_ => $"{alerts.Count} drift alerts detected. Review SBOM and vulnerability parity."
|
||||
};
|
||||
|
||||
return new DriftAnalysisResult
|
||||
{
|
||||
AnalyzedAt = DateTime.UtcNow,
|
||||
ResultCount = results.Count,
|
||||
Alerts = alerts,
|
||||
Summary = summary
|
||||
};
|
||||
}
|
||||
|
||||
private static DriftCalculation CalculateDrift(IReadOnlyList<(DateTime Timestamp, double Value)> series)
|
||||
{
|
||||
if (series.Count < 2)
|
||||
{
|
||||
return new DriftCalculation { HasDrift = false };
|
||||
}
|
||||
|
||||
// Split into first half (baseline) and second half (current)
|
||||
var midpoint = series.Count / 2;
|
||||
var baseline = series.Take(midpoint).Select(x => x.Value).ToList();
|
||||
var current = series.Skip(midpoint).Select(x => x.Value).ToList();
|
||||
|
||||
var baselineAvg = baseline.Average();
|
||||
var currentAvg = current.Average();
|
||||
|
||||
// Drift is relative decline: (baseline - current) / baseline
|
||||
if (baselineAvg <= 0)
|
||||
{
|
||||
return new DriftCalculation { HasDrift = false };
|
||||
}
|
||||
|
||||
var drift = (baselineAvg - currentAvg) / baselineAvg;
|
||||
|
||||
return new DriftCalculation
|
||||
{
|
||||
HasDrift = drift > 0, // Only flag declining metrics
|
||||
BaselineValue = baselineAvg,
|
||||
CurrentValue = currentAvg,
|
||||
DriftAmount = Math.Max(0, drift)
|
||||
};
|
||||
}
|
||||
|
||||
private static DriftCalculation CalculateInverseDrift(IReadOnlyList<(DateTime Timestamp, double Value)> series)
|
||||
{
|
||||
if (series.Count < 2)
|
||||
{
|
||||
return new DriftCalculation { HasDrift = false };
|
||||
}
|
||||
|
||||
// Split into first half (baseline) and second half (current)
|
||||
var midpoint = series.Count / 2;
|
||||
var baseline = series.Take(midpoint).Select(x => x.Value).ToList();
|
||||
var current = series.Skip(midpoint).Select(x => x.Value).ToList();
|
||||
|
||||
var baselineAvg = baseline.Average();
|
||||
var currentAvg = current.Average();
|
||||
|
||||
// Drift is relative increase: (current - baseline) / baseline
|
||||
if (baselineAvg <= 0)
|
||||
{
|
||||
return new DriftCalculation { HasDrift = false };
|
||||
}
|
||||
|
||||
var drift = (currentAvg - baselineAvg) / baselineAvg;
|
||||
|
||||
return new DriftCalculation
|
||||
{
|
||||
HasDrift = drift > 0, // Only flag increasing metrics (bad for latency)
|
||||
BaselineValue = baselineAvg,
|
||||
CurrentValue = currentAvg,
|
||||
DriftAmount = Math.Max(0, drift)
|
||||
};
|
||||
}
|
||||
|
||||
private static AlertSeverity GetAlertSeverity(double drift, double threshold)
|
||||
{
|
||||
var ratio = drift / threshold;
|
||||
|
||||
return ratio switch
|
||||
{
|
||||
>= 3.0 => AlertSeverity.Critical,
|
||||
>= 2.0 => AlertSeverity.High,
|
||||
>= 1.5 => AlertSeverity.Medium,
|
||||
_ => AlertSeverity.Low
|
||||
};
|
||||
}
|
||||
|
||||
private sealed record DriftCalculation
|
||||
{
|
||||
public bool HasDrift { get; init; }
|
||||
public double BaselineValue { get; init; }
|
||||
public double CurrentValue { get; init; }
|
||||
public double DriftAmount { get; init; }
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Thresholds for triggering drift alerts.
|
||||
/// Values are relative (e.g., 0.05 = 5% drift).
|
||||
/// </summary>
|
||||
public sealed record DriftThresholds
|
||||
{
|
||||
/// <summary>
|
||||
/// Default thresholds: 5% drift on key metrics.
|
||||
/// </summary>
|
||||
public static DriftThresholds Default => new()
|
||||
{
|
||||
SbomCompletenessThreshold = 0.05,
|
||||
PurlCompletenessThreshold = 0.05,
|
||||
VulnRecallThreshold = 0.05,
|
||||
F1ScoreThreshold = 0.05,
|
||||
LatencyRatioThreshold = 0.10 // 10% for latency
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// SBOM package completeness drift threshold (default: 5%).
|
||||
/// </summary>
|
||||
public double SbomCompletenessThreshold { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// PURL completeness drift threshold (default: 5%).
|
||||
/// </summary>
|
||||
public double PurlCompletenessThreshold { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Vulnerability recall drift threshold (default: 5%).
|
||||
/// </summary>
|
||||
public double VulnRecallThreshold { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Vulnerability F1 score drift threshold (default: 5%).
|
||||
/// </summary>
|
||||
public double F1ScoreThreshold { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Latency ratio drift threshold (default: 10%).
|
||||
/// </summary>
|
||||
public double LatencyRatioThreshold { get; init; }
|
||||
}
|
||||
|
||||
public sealed record DriftAnalysisResult
|
||||
{
|
||||
public required DateTime AnalyzedAt { get; init; }
|
||||
public required int ResultCount { get; init; }
|
||||
public required IReadOnlyList<DriftAlert> Alerts { get; init; }
|
||||
public required string Summary { get; init; }
|
||||
|
||||
public bool HasAlerts => Alerts.Count > 0;
|
||||
public bool HasCriticalAlerts => Alerts.Any(a => a.Severity == AlertSeverity.Critical);
|
||||
}
|
||||
|
||||
public sealed record DriftAlert
|
||||
{
|
||||
public required string MetricName { get; init; }
|
||||
public required DriftType DriftType { get; init; }
|
||||
public required double BaselineValue { get; init; }
|
||||
public required double CurrentValue { get; init; }
|
||||
public required double DriftPercent { get; init; }
|
||||
public required double ThresholdPercent { get; init; }
|
||||
public required int TrendDays { get; init; }
|
||||
public required AlertSeverity Severity { get; init; }
|
||||
public required string Recommendation { get; init; }
|
||||
}
|
||||
|
||||
public enum DriftType
|
||||
{
|
||||
Declining,
|
||||
Increasing
|
||||
}
|
||||
|
||||
public enum AlertSeverity
|
||||
{
|
||||
Low,
|
||||
Medium,
|
||||
High,
|
||||
Critical
|
||||
}
|
||||
296
tests/parity/StellaOps.Parity.Tests/Storage/ParityResultStore.cs
Normal file
296
tests/parity/StellaOps.Parity.Tests/Storage/ParityResultStore.cs
Normal file
@@ -0,0 +1,296 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
// SPDX-FileCopyrightText: 2025 StellaOps Contributors
|
||||
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Parity.Tests.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Time-series storage for parity test results.
|
||||
/// Emits results as JSON artifacts for historical tracking and trend analysis.
|
||||
/// </summary>
|
||||
public sealed class ParityResultStore
|
||||
{
|
||||
private readonly string _storagePath;
|
||||
private readonly JsonSerializerOptions _jsonOptions;
|
||||
|
||||
public ParityResultStore(string storagePath)
|
||||
{
|
||||
_storagePath = storagePath ?? throw new ArgumentNullException(nameof(storagePath));
|
||||
|
||||
_jsonOptions = new JsonSerializerOptions
|
||||
{
|
||||
WriteIndented = true,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
Converters = { new JsonStringEnumConverter() }
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stores a parity run result with timestamp-based filename.
|
||||
/// </summary>
|
||||
public async Task StoreResultAsync(ParityRunSummary summary, CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(summary);
|
||||
|
||||
Directory.CreateDirectory(_storagePath);
|
||||
|
||||
var filename = $"parity-{summary.RunId}-{summary.Timestamp:yyyyMMddTHHmmssZ}.json";
|
||||
var filePath = Path.Combine(_storagePath, filename);
|
||||
|
||||
var json = JsonSerializer.Serialize(summary, _jsonOptions);
|
||||
await File.WriteAllTextAsync(filePath, json, ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads all parity results within the specified time range.
|
||||
/// </summary>
|
||||
public async Task<IReadOnlyList<ParityRunSummary>> LoadResultsAsync(
|
||||
DateTime? startTime = null,
|
||||
DateTime? endTime = null,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
if (!Directory.Exists(_storagePath))
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var results = new List<ParityRunSummary>();
|
||||
var files = Directory.GetFiles(_storagePath, "parity-*.json");
|
||||
|
||||
foreach (var file in files)
|
||||
{
|
||||
try
|
||||
{
|
||||
var json = await File.ReadAllTextAsync(file, ct);
|
||||
var summary = JsonSerializer.Deserialize<ParityRunSummary>(json, _jsonOptions);
|
||||
|
||||
if (summary is null)
|
||||
continue;
|
||||
|
||||
// Filter by time range
|
||||
if (startTime.HasValue && summary.Timestamp < startTime.Value)
|
||||
continue;
|
||||
|
||||
if (endTime.HasValue && summary.Timestamp > endTime.Value)
|
||||
continue;
|
||||
|
||||
results.Add(summary);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
// Skip malformed files
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
.OrderBy(r => r.Timestamp)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads the latest N results.
|
||||
/// </summary>
|
||||
public async Task<IReadOnlyList<ParityRunSummary>> LoadLatestResultsAsync(
|
||||
int count,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
var allResults = await LoadResultsAsync(ct: ct);
|
||||
return allResults
|
||||
.OrderByDescending(r => r.Timestamp)
|
||||
.Take(count)
|
||||
.OrderBy(r => r.Timestamp)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Prunes results older than the specified retention period.
|
||||
/// </summary>
|
||||
public async Task<int> PruneOldResultsAsync(TimeSpan retention, CancellationToken ct = default)
|
||||
{
|
||||
var cutoff = DateTime.UtcNow - retention;
|
||||
var results = await LoadResultsAsync(endTime: cutoff, ct: ct);
|
||||
|
||||
var pruned = 0;
|
||||
foreach (var result in results)
|
||||
{
|
||||
var filename = $"parity-{result.RunId}-{result.Timestamp:yyyyMMddTHHmmssZ}.json";
|
||||
var filePath = Path.Combine(_storagePath, filename);
|
||||
|
||||
if (File.Exists(filePath))
|
||||
{
|
||||
File.Delete(filePath);
|
||||
pruned++;
|
||||
}
|
||||
}
|
||||
|
||||
return pruned;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Exports results to Prometheus-compatible metrics format (text).
|
||||
/// </summary>
|
||||
public string ExportPrometheusMetrics(ParityRunSummary summary)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(summary);
|
||||
|
||||
var lines = new List<string>
|
||||
{
|
||||
"# HELP stellaops_parity_sbom_completeness_ratio SBOM package completeness ratio vs competitors.",
|
||||
"# TYPE stellaops_parity_sbom_completeness_ratio gauge",
|
||||
$"stellaops_parity_sbom_completeness_ratio{{run_id=\"{summary.RunId}\"}} {summary.SbomMetrics.PackageCompletenessRatio:F4}",
|
||||
"",
|
||||
"# HELP stellaops_parity_sbom_match_count Number of matched packages across scanners.",
|
||||
"# TYPE stellaops_parity_sbom_match_count gauge",
|
||||
$"stellaops_parity_sbom_match_count{{run_id=\"{summary.RunId}\"}} {summary.SbomMetrics.MatchedPackageCount}",
|
||||
"",
|
||||
"# HELP stellaops_parity_vuln_recall Vulnerability detection recall vs competitors.",
|
||||
"# TYPE stellaops_parity_vuln_recall gauge",
|
||||
$"stellaops_parity_vuln_recall{{run_id=\"{summary.RunId}\"}} {summary.VulnMetrics.Recall:F4}",
|
||||
"",
|
||||
"# HELP stellaops_parity_vuln_precision Vulnerability detection precision vs competitors.",
|
||||
"# TYPE stellaops_parity_vuln_precision gauge",
|
||||
$"stellaops_parity_vuln_precision{{run_id=\"{summary.RunId}\"}} {summary.VulnMetrics.Precision:F4}",
|
||||
"",
|
||||
"# HELP stellaops_parity_vuln_f1 Vulnerability detection F1 score.",
|
||||
"# TYPE stellaops_parity_vuln_f1 gauge",
|
||||
$"stellaops_parity_vuln_f1{{run_id=\"{summary.RunId}\"}} {summary.VulnMetrics.F1Score:F4}",
|
||||
"",
|
||||
"# HELP stellaops_parity_latency_p50_ms Scan latency P50 in milliseconds.",
|
||||
"# TYPE stellaops_parity_latency_p50_ms gauge",
|
||||
$"stellaops_parity_latency_p50_ms{{scanner=\"stellaops\",run_id=\"{summary.RunId}\"}} {summary.LatencyMetrics.StellaOpsP50Ms:F2}",
|
||||
$"stellaops_parity_latency_p50_ms{{scanner=\"grype\",run_id=\"{summary.RunId}\"}} {summary.LatencyMetrics.GrypeP50Ms:F2}",
|
||||
$"stellaops_parity_latency_p50_ms{{scanner=\"trivy\",run_id=\"{summary.RunId}\"}} {summary.LatencyMetrics.TrivyP50Ms:F2}",
|
||||
"",
|
||||
"# HELP stellaops_parity_latency_p95_ms Scan latency P95 in milliseconds.",
|
||||
"# TYPE stellaops_parity_latency_p95_ms gauge",
|
||||
$"stellaops_parity_latency_p95_ms{{scanner=\"stellaops\",run_id=\"{summary.RunId}\"}} {summary.LatencyMetrics.StellaOpsP95Ms:F2}",
|
||||
$"stellaops_parity_latency_p95_ms{{scanner=\"grype\",run_id=\"{summary.RunId}\"}} {summary.LatencyMetrics.GrypeP95Ms:F2}",
|
||||
$"stellaops_parity_latency_p95_ms{{scanner=\"trivy\",run_id=\"{summary.RunId}\"}} {summary.LatencyMetrics.TrivyP95Ms:F2}",
|
||||
"",
|
||||
"# HELP stellaops_parity_error_scenarios_passed Number of error scenarios passed.",
|
||||
"# TYPE stellaops_parity_error_scenarios_passed gauge",
|
||||
$"stellaops_parity_error_scenarios_passed{{run_id=\"{summary.RunId}\"}} {summary.ErrorMetrics.ScenariosPassed}",
|
||||
"",
|
||||
"# HELP stellaops_parity_error_scenarios_total Total number of error scenarios.",
|
||||
"# TYPE stellaops_parity_error_scenarios_total gauge",
|
||||
$"stellaops_parity_error_scenarios_total{{run_id=\"{summary.RunId}\"}} {summary.ErrorMetrics.ScenariosTotal}"
|
||||
};
|
||||
|
||||
return string.Join("\n", lines);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Exports results to InfluxDB line protocol format.
|
||||
/// </summary>
|
||||
public string ExportInfluxLineProtocol(ParityRunSummary summary)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(summary);
|
||||
|
||||
var timestamp = new DateTimeOffset(summary.Timestamp).ToUnixTimeMilliseconds() * 1_000_000; // nanoseconds
|
||||
var lines = new List<string>
|
||||
{
|
||||
$"parity_sbom,run_id={summary.RunId} completeness_ratio={summary.SbomMetrics.PackageCompletenessRatio:F4},matched_count={summary.SbomMetrics.MatchedPackageCount}i {timestamp}",
|
||||
$"parity_vuln,run_id={summary.RunId} recall={summary.VulnMetrics.Recall:F4},precision={summary.VulnMetrics.Precision:F4},f1={summary.VulnMetrics.F1Score:F4} {timestamp}",
|
||||
$"parity_latency,run_id={summary.RunId},scanner=stellaops p50={summary.LatencyMetrics.StellaOpsP50Ms:F2},p95={summary.LatencyMetrics.StellaOpsP95Ms:F2},p99={summary.LatencyMetrics.StellaOpsP99Ms:F2} {timestamp}",
|
||||
$"parity_latency,run_id={summary.RunId},scanner=grype p50={summary.LatencyMetrics.GrypeP50Ms:F2},p95={summary.LatencyMetrics.GrypeP95Ms:F2},p99={summary.LatencyMetrics.GrypeP99Ms:F2} {timestamp}",
|
||||
$"parity_latency,run_id={summary.RunId},scanner=trivy p50={summary.LatencyMetrics.TrivyP50Ms:F2},p95={summary.LatencyMetrics.TrivyP95Ms:F2},p99={summary.LatencyMetrics.TrivyP99Ms:F2} {timestamp}",
|
||||
$"parity_errors,run_id={summary.RunId} passed={summary.ErrorMetrics.ScenariosPassed}i,total={summary.ErrorMetrics.ScenariosTotal}i {timestamp}"
|
||||
};
|
||||
|
||||
return string.Join("\n", lines);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Summary of a single parity test run.
|
||||
/// </summary>
|
||||
public sealed record ParityRunSummary
|
||||
{
|
||||
public required string RunId { get; init; }
|
||||
public required DateTime Timestamp { get; init; }
|
||||
public required string StellaOpsVersion { get; init; }
|
||||
public required CompetitorVersions CompetitorVersions { get; init; }
|
||||
public required SbomParityMetrics SbomMetrics { get; init; }
|
||||
public required VulnParityMetrics VulnMetrics { get; init; }
|
||||
public required LatencyParityMetrics LatencyMetrics { get; init; }
|
||||
public required ErrorParityMetrics ErrorMetrics { get; init; }
|
||||
public required IReadOnlyList<string> FixturesUsed { get; init; }
|
||||
public string? Notes { get; init; }
|
||||
}
|
||||
|
||||
public sealed record CompetitorVersions
|
||||
{
|
||||
public required string SyftVersion { get; init; }
|
||||
public required string GrypeVersion { get; init; }
|
||||
public required string TrivyVersion { get; init; }
|
||||
}
|
||||
|
||||
public sealed record SbomParityMetrics
|
||||
{
|
||||
public required int StellaOpsPackageCount { get; init; }
|
||||
public required int SyftPackageCount { get; init; }
|
||||
public required int MatchedPackageCount { get; init; }
|
||||
public required double PackageCompletenessRatio { get; init; }
|
||||
public required double PurlCompletenessRatio { get; init; }
|
||||
public required double LicenseDetectionRatio { get; init; }
|
||||
public required double CpeDetectionRatio { get; init; }
|
||||
}
|
||||
|
||||
public sealed record VulnParityMetrics
|
||||
{
|
||||
public required int StellaOpsCveCount { get; init; }
|
||||
public required int GrypeCveCount { get; init; }
|
||||
public required int TrivyCveCount { get; init; }
|
||||
public required int UnionCveCount { get; init; }
|
||||
public required int IntersectionCveCount { get; init; }
|
||||
public required double Recall { get; init; }
|
||||
public required double Precision { get; init; }
|
||||
public required double F1Score { get; init; }
|
||||
public required SeverityBreakdown StellaOpsSeverity { get; init; }
|
||||
public required SeverityBreakdown GrypeSeverity { get; init; }
|
||||
public required SeverityBreakdown TrivySeverity { get; init; }
|
||||
}
|
||||
|
||||
public sealed record SeverityBreakdown
|
||||
{
|
||||
public int Critical { get; init; }
|
||||
public int High { get; init; }
|
||||
public int Medium { get; init; }
|
||||
public int Low { get; init; }
|
||||
public int Unknown { get; init; }
|
||||
}
|
||||
|
||||
public sealed record LatencyParityMetrics
|
||||
{
|
||||
public required double StellaOpsP50Ms { get; init; }
|
||||
public required double StellaOpsP95Ms { get; init; }
|
||||
public required double StellaOpsP99Ms { get; init; }
|
||||
public required double GrypeP50Ms { get; init; }
|
||||
public required double GrypeP95Ms { get; init; }
|
||||
public required double GrypeP99Ms { get; init; }
|
||||
public required double TrivyP50Ms { get; init; }
|
||||
public required double TrivyP95Ms { get; init; }
|
||||
public required double TrivyP99Ms { get; init; }
|
||||
public required double StellaOpsVsGrypeRatio { get; init; }
|
||||
public required double StellaOpsVsTrivyRatio { get; init; }
|
||||
}
|
||||
|
||||
public sealed record ErrorParityMetrics
|
||||
{
|
||||
public required int ScenariosPassed { get; init; }
|
||||
public required int ScenariosTotal { get; init; }
|
||||
public required IReadOnlyList<ErrorScenarioResult> ScenarioResults { get; init; }
|
||||
}
|
||||
|
||||
public sealed record ErrorScenarioResult
|
||||
{
|
||||
public required string ScenarioName { get; init; }
|
||||
public required bool StellaOpsPassed { get; init; }
|
||||
public required bool GrypePassed { get; init; }
|
||||
public required bool TrivyPassed { get; init; }
|
||||
public required bool MatchesExpected { get; init; }
|
||||
public string? Notes { get; init; }
|
||||
}
|
||||
Reference in New Issue
Block a user