// SPDX-License-Identifier: AGPL-3.0-or-later // SPDX-FileCopyrightText: 2025 StellaOps Contributors using System.Text.Json; using System.Text.Json.Serialization; namespace StellaOps.Parity.Tests.Storage; /// /// Time-series storage for parity test results. /// Emits results as JSON artifacts for historical tracking and trend analysis. /// public sealed class ParityResultStore { private readonly string _storagePath; private readonly JsonSerializerOptions _jsonOptions; public ParityResultStore(string storagePath) { _storagePath = storagePath ?? throw new ArgumentNullException(nameof(storagePath)); _jsonOptions = new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, Converters = { new JsonStringEnumConverter() } }; } /// /// Stores a parity run result with timestamp-based filename. /// public async Task StoreResultAsync(ParityRunSummary summary, CancellationToken ct = default) { ArgumentNullException.ThrowIfNull(summary); Directory.CreateDirectory(_storagePath); var filename = $"parity-{summary.RunId}-{summary.Timestamp:yyyyMMddTHHmmssZ}.json"; var filePath = Path.Combine(_storagePath, filename); var json = JsonSerializer.Serialize(summary, _jsonOptions); await File.WriteAllTextAsync(filePath, json, ct); } /// /// Loads all parity results within the specified time range. /// public async Task> LoadResultsAsync( DateTime? startTime = null, DateTime? endTime = null, CancellationToken ct = default) { if (!Directory.Exists(_storagePath)) { return []; } var results = new List(); var files = Directory.GetFiles(_storagePath, "parity-*.json"); foreach (var file in files) { try { var json = await File.ReadAllTextAsync(file, ct); var summary = JsonSerializer.Deserialize(json, _jsonOptions); if (summary is null) continue; // Filter by time range if (startTime.HasValue && summary.Timestamp < startTime.Value) continue; if (endTime.HasValue && summary.Timestamp > endTime.Value) continue; results.Add(summary); } catch (JsonException) { // Skip malformed files } } return results .OrderBy(r => r.Timestamp) .ToList(); } /// /// Loads the latest N results. /// public async Task> LoadLatestResultsAsync( int count, CancellationToken ct = default) { var allResults = await LoadResultsAsync(ct: ct); return allResults .OrderByDescending(r => r.Timestamp) .Take(count) .OrderBy(r => r.Timestamp) .ToList(); } /// /// Prunes results older than the specified retention period. /// public async Task PruneOldResultsAsync(TimeSpan retention, CancellationToken ct = default) { var cutoff = DateTime.UtcNow - retention; var results = await LoadResultsAsync(endTime: cutoff, ct: ct); var pruned = 0; foreach (var result in results) { var filename = $"parity-{result.RunId}-{result.Timestamp:yyyyMMddTHHmmssZ}.json"; var filePath = Path.Combine(_storagePath, filename); if (File.Exists(filePath)) { File.Delete(filePath); pruned++; } } return pruned; } /// /// Exports results to Prometheus-compatible metrics format (text). /// public string ExportPrometheusMetrics(ParityRunSummary summary) { ArgumentNullException.ThrowIfNull(summary); var lines = new List { "# HELP stellaops_parity_sbom_completeness_ratio SBOM package completeness ratio vs competitors.", "# TYPE stellaops_parity_sbom_completeness_ratio gauge", $"stellaops_parity_sbom_completeness_ratio{{run_id=\"{summary.RunId}\"}} {summary.SbomMetrics.PackageCompletenessRatio:F4}", "", "# HELP stellaops_parity_sbom_match_count Number of matched packages across scanners.", "# TYPE stellaops_parity_sbom_match_count gauge", $"stellaops_parity_sbom_match_count{{run_id=\"{summary.RunId}\"}} {summary.SbomMetrics.MatchedPackageCount}", "", "# HELP stellaops_parity_vuln_recall Vulnerability detection recall vs competitors.", "# TYPE stellaops_parity_vuln_recall gauge", $"stellaops_parity_vuln_recall{{run_id=\"{summary.RunId}\"}} {summary.VulnMetrics.Recall:F4}", "", "# HELP stellaops_parity_vuln_precision Vulnerability detection precision vs competitors.", "# TYPE stellaops_parity_vuln_precision gauge", $"stellaops_parity_vuln_precision{{run_id=\"{summary.RunId}\"}} {summary.VulnMetrics.Precision:F4}", "", "# HELP stellaops_parity_vuln_f1 Vulnerability detection F1 score.", "# TYPE stellaops_parity_vuln_f1 gauge", $"stellaops_parity_vuln_f1{{run_id=\"{summary.RunId}\"}} {summary.VulnMetrics.F1Score:F4}", "", "# HELP stellaops_parity_latency_p50_ms Scan latency P50 in milliseconds.", "# TYPE stellaops_parity_latency_p50_ms gauge", $"stellaops_parity_latency_p50_ms{{scanner=\"stellaops\",run_id=\"{summary.RunId}\"}} {summary.LatencyMetrics.StellaOpsP50Ms:F2}", $"stellaops_parity_latency_p50_ms{{scanner=\"grype\",run_id=\"{summary.RunId}\"}} {summary.LatencyMetrics.GrypeP50Ms:F2}", $"stellaops_parity_latency_p50_ms{{scanner=\"trivy\",run_id=\"{summary.RunId}\"}} {summary.LatencyMetrics.TrivyP50Ms:F2}", "", "# HELP stellaops_parity_latency_p95_ms Scan latency P95 in milliseconds.", "# TYPE stellaops_parity_latency_p95_ms gauge", $"stellaops_parity_latency_p95_ms{{scanner=\"stellaops\",run_id=\"{summary.RunId}\"}} {summary.LatencyMetrics.StellaOpsP95Ms:F2}", $"stellaops_parity_latency_p95_ms{{scanner=\"grype\",run_id=\"{summary.RunId}\"}} {summary.LatencyMetrics.GrypeP95Ms:F2}", $"stellaops_parity_latency_p95_ms{{scanner=\"trivy\",run_id=\"{summary.RunId}\"}} {summary.LatencyMetrics.TrivyP95Ms:F2}", "", "# HELP stellaops_parity_error_scenarios_passed Number of error scenarios passed.", "# TYPE stellaops_parity_error_scenarios_passed gauge", $"stellaops_parity_error_scenarios_passed{{run_id=\"{summary.RunId}\"}} {summary.ErrorMetrics.ScenariosPassed}", "", "# HELP stellaops_parity_error_scenarios_total Total number of error scenarios.", "# TYPE stellaops_parity_error_scenarios_total gauge", $"stellaops_parity_error_scenarios_total{{run_id=\"{summary.RunId}\"}} {summary.ErrorMetrics.ScenariosTotal}" }; return string.Join("\n", lines); } /// /// Exports results to InfluxDB line protocol format. /// public string ExportInfluxLineProtocol(ParityRunSummary summary) { ArgumentNullException.ThrowIfNull(summary); var timestamp = new DateTimeOffset(summary.Timestamp).ToUnixTimeMilliseconds() * 1_000_000; // nanoseconds var lines = new List { $"parity_sbom,run_id={summary.RunId} completeness_ratio={summary.SbomMetrics.PackageCompletenessRatio:F4},matched_count={summary.SbomMetrics.MatchedPackageCount}i {timestamp}", $"parity_vuln,run_id={summary.RunId} recall={summary.VulnMetrics.Recall:F4},precision={summary.VulnMetrics.Precision:F4},f1={summary.VulnMetrics.F1Score:F4} {timestamp}", $"parity_latency,run_id={summary.RunId},scanner=stellaops p50={summary.LatencyMetrics.StellaOpsP50Ms:F2},p95={summary.LatencyMetrics.StellaOpsP95Ms:F2},p99={summary.LatencyMetrics.StellaOpsP99Ms:F2} {timestamp}", $"parity_latency,run_id={summary.RunId},scanner=grype p50={summary.LatencyMetrics.GrypeP50Ms:F2},p95={summary.LatencyMetrics.GrypeP95Ms:F2},p99={summary.LatencyMetrics.GrypeP99Ms:F2} {timestamp}", $"parity_latency,run_id={summary.RunId},scanner=trivy p50={summary.LatencyMetrics.TrivyP50Ms:F2},p95={summary.LatencyMetrics.TrivyP95Ms:F2},p99={summary.LatencyMetrics.TrivyP99Ms:F2} {timestamp}", $"parity_errors,run_id={summary.RunId} passed={summary.ErrorMetrics.ScenariosPassed}i,total={summary.ErrorMetrics.ScenariosTotal}i {timestamp}" }; return string.Join("\n", lines); } } /// /// Summary of a single parity test run. /// public sealed record ParityRunSummary { public required string RunId { get; init; } public required DateTime Timestamp { get; init; } public required string StellaOpsVersion { get; init; } public required CompetitorVersions CompetitorVersions { get; init; } public required SbomParityMetrics SbomMetrics { get; init; } public required VulnParityMetrics VulnMetrics { get; init; } public required LatencyParityMetrics LatencyMetrics { get; init; } public required ErrorParityMetrics ErrorMetrics { get; init; } public required IReadOnlyList FixturesUsed { get; init; } public string? Notes { get; init; } } public sealed record CompetitorVersions { public required string SyftVersion { get; init; } public required string GrypeVersion { get; init; } public required string TrivyVersion { get; init; } } public sealed record SbomParityMetrics { public required int StellaOpsPackageCount { get; init; } public required int SyftPackageCount { get; init; } public required int MatchedPackageCount { get; init; } public required double PackageCompletenessRatio { get; init; } public required double PurlCompletenessRatio { get; init; } public required double LicenseDetectionRatio { get; init; } public required double CpeDetectionRatio { get; init; } } public sealed record VulnParityMetrics { public required int StellaOpsCveCount { get; init; } public required int GrypeCveCount { get; init; } public required int TrivyCveCount { get; init; } public required int UnionCveCount { get; init; } public required int IntersectionCveCount { get; init; } public required double Recall { get; init; } public required double Precision { get; init; } public required double F1Score { get; init; } public required SeverityBreakdown StellaOpsSeverity { get; init; } public required SeverityBreakdown GrypeSeverity { get; init; } public required SeverityBreakdown TrivySeverity { get; init; } } public sealed record SeverityBreakdown { public int Critical { get; init; } public int High { get; init; } public int Medium { get; init; } public int Low { get; init; } public int Unknown { get; init; } } public sealed record LatencyParityMetrics { public required double StellaOpsP50Ms { get; init; } public required double StellaOpsP95Ms { get; init; } public required double StellaOpsP99Ms { get; init; } public required double GrypeP50Ms { get; init; } public required double GrypeP95Ms { get; init; } public required double GrypeP99Ms { get; init; } public required double TrivyP50Ms { get; init; } public required double TrivyP95Ms { get; init; } public required double TrivyP99Ms { get; init; } public required double StellaOpsVsGrypeRatio { get; init; } public required double StellaOpsVsTrivyRatio { get; init; } } public sealed record ErrorParityMetrics { public required int ScenariosPassed { get; init; } public required int ScenariosTotal { get; init; } public required IReadOnlyList ScenarioResults { get; init; } } public sealed record ErrorScenarioResult { public required string ScenarioName { get; init; } public required bool StellaOpsPassed { get; init; } public required bool GrypePassed { get; init; } public required bool TrivyPassed { get; init; } public required bool MatchesExpected { get; init; } public string? Notes { get; init; } }