297 lines
12 KiB
C#
297 lines
12 KiB
C#
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
// SPDX-FileCopyrightText: 2025 StellaOps Contributors
|
|
|
|
using System.Text.Json;
|
|
using System.Text.Json.Serialization;
|
|
|
|
namespace StellaOps.Parity.Tests.Storage;
|
|
|
|
/// <summary>
|
|
/// Time-series storage for parity test results.
|
|
/// Emits results as JSON artifacts for historical tracking and trend analysis.
|
|
/// </summary>
|
|
public sealed class ParityResultStore
|
|
{
|
|
private readonly string _storagePath;
|
|
private readonly JsonSerializerOptions _jsonOptions;
|
|
|
|
public ParityResultStore(string storagePath)
|
|
{
|
|
_storagePath = storagePath ?? throw new ArgumentNullException(nameof(storagePath));
|
|
|
|
_jsonOptions = new JsonSerializerOptions
|
|
{
|
|
WriteIndented = true,
|
|
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
|
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
|
Converters = { new JsonStringEnumConverter() }
|
|
};
|
|
}
|
|
|
|
/// <summary>
|
|
/// Stores a parity run result with timestamp-based filename.
|
|
/// </summary>
|
|
public async Task StoreResultAsync(ParityRunSummary summary, CancellationToken ct = default)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(summary);
|
|
|
|
Directory.CreateDirectory(_storagePath);
|
|
|
|
var filename = $"parity-{summary.RunId}-{summary.Timestamp:yyyyMMddTHHmmssZ}.json";
|
|
var filePath = Path.Combine(_storagePath, filename);
|
|
|
|
var json = JsonSerializer.Serialize(summary, _jsonOptions);
|
|
await File.WriteAllTextAsync(filePath, json, ct);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Loads all parity results within the specified time range.
|
|
/// </summary>
|
|
public async Task<IReadOnlyList<ParityRunSummary>> LoadResultsAsync(
|
|
DateTime? startTime = null,
|
|
DateTime? endTime = null,
|
|
CancellationToken ct = default)
|
|
{
|
|
if (!Directory.Exists(_storagePath))
|
|
{
|
|
return [];
|
|
}
|
|
|
|
var results = new List<ParityRunSummary>();
|
|
var files = Directory.GetFiles(_storagePath, "parity-*.json");
|
|
|
|
foreach (var file in files)
|
|
{
|
|
try
|
|
{
|
|
var json = await File.ReadAllTextAsync(file, ct);
|
|
var summary = JsonSerializer.Deserialize<ParityRunSummary>(json, _jsonOptions);
|
|
|
|
if (summary is null)
|
|
continue;
|
|
|
|
// Filter by time range
|
|
if (startTime.HasValue && summary.Timestamp < startTime.Value)
|
|
continue;
|
|
|
|
if (endTime.HasValue && summary.Timestamp > endTime.Value)
|
|
continue;
|
|
|
|
results.Add(summary);
|
|
}
|
|
catch (JsonException)
|
|
{
|
|
// Skip malformed files
|
|
}
|
|
}
|
|
|
|
return results
|
|
.OrderBy(r => r.Timestamp)
|
|
.ToList();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Loads the latest N results.
|
|
/// </summary>
|
|
public async Task<IReadOnlyList<ParityRunSummary>> LoadLatestResultsAsync(
|
|
int count,
|
|
CancellationToken ct = default)
|
|
{
|
|
var allResults = await LoadResultsAsync(ct: ct);
|
|
return allResults
|
|
.OrderByDescending(r => r.Timestamp)
|
|
.Take(count)
|
|
.OrderBy(r => r.Timestamp)
|
|
.ToList();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Prunes results older than the specified retention period.
|
|
/// </summary>
|
|
public async Task<int> PruneOldResultsAsync(TimeSpan retention, CancellationToken ct = default)
|
|
{
|
|
var cutoff = DateTime.UtcNow - retention;
|
|
var results = await LoadResultsAsync(endTime: cutoff, ct: ct);
|
|
|
|
var pruned = 0;
|
|
foreach (var result in results)
|
|
{
|
|
var filename = $"parity-{result.RunId}-{result.Timestamp:yyyyMMddTHHmmssZ}.json";
|
|
var filePath = Path.Combine(_storagePath, filename);
|
|
|
|
if (File.Exists(filePath))
|
|
{
|
|
File.Delete(filePath);
|
|
pruned++;
|
|
}
|
|
}
|
|
|
|
return pruned;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Exports results to Prometheus-compatible metrics format (text).
|
|
/// </summary>
|
|
public string ExportPrometheusMetrics(ParityRunSummary summary)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(summary);
|
|
|
|
var lines = new List<string>
|
|
{
|
|
"# HELP stellaops_parity_sbom_completeness_ratio SBOM package completeness ratio vs competitors.",
|
|
"# TYPE stellaops_parity_sbom_completeness_ratio gauge",
|
|
$"stellaops_parity_sbom_completeness_ratio{{run_id=\"{summary.RunId}\"}} {summary.SbomMetrics.PackageCompletenessRatio:F4}",
|
|
"",
|
|
"# HELP stellaops_parity_sbom_match_count Number of matched packages across scanners.",
|
|
"# TYPE stellaops_parity_sbom_match_count gauge",
|
|
$"stellaops_parity_sbom_match_count{{run_id=\"{summary.RunId}\"}} {summary.SbomMetrics.MatchedPackageCount}",
|
|
"",
|
|
"# HELP stellaops_parity_vuln_recall Vulnerability detection recall vs competitors.",
|
|
"# TYPE stellaops_parity_vuln_recall gauge",
|
|
$"stellaops_parity_vuln_recall{{run_id=\"{summary.RunId}\"}} {summary.VulnMetrics.Recall:F4}",
|
|
"",
|
|
"# HELP stellaops_parity_vuln_precision Vulnerability detection precision vs competitors.",
|
|
"# TYPE stellaops_parity_vuln_precision gauge",
|
|
$"stellaops_parity_vuln_precision{{run_id=\"{summary.RunId}\"}} {summary.VulnMetrics.Precision:F4}",
|
|
"",
|
|
"# HELP stellaops_parity_vuln_f1 Vulnerability detection F1 score.",
|
|
"# TYPE stellaops_parity_vuln_f1 gauge",
|
|
$"stellaops_parity_vuln_f1{{run_id=\"{summary.RunId}\"}} {summary.VulnMetrics.F1Score:F4}",
|
|
"",
|
|
"# HELP stellaops_parity_latency_p50_ms Scan latency P50 in milliseconds.",
|
|
"# TYPE stellaops_parity_latency_p50_ms gauge",
|
|
$"stellaops_parity_latency_p50_ms{{scanner=\"stellaops\",run_id=\"{summary.RunId}\"}} {summary.LatencyMetrics.StellaOpsP50Ms:F2}",
|
|
$"stellaops_parity_latency_p50_ms{{scanner=\"grype\",run_id=\"{summary.RunId}\"}} {summary.LatencyMetrics.GrypeP50Ms:F2}",
|
|
$"stellaops_parity_latency_p50_ms{{scanner=\"trivy\",run_id=\"{summary.RunId}\"}} {summary.LatencyMetrics.TrivyP50Ms:F2}",
|
|
"",
|
|
"# HELP stellaops_parity_latency_p95_ms Scan latency P95 in milliseconds.",
|
|
"# TYPE stellaops_parity_latency_p95_ms gauge",
|
|
$"stellaops_parity_latency_p95_ms{{scanner=\"stellaops\",run_id=\"{summary.RunId}\"}} {summary.LatencyMetrics.StellaOpsP95Ms:F2}",
|
|
$"stellaops_parity_latency_p95_ms{{scanner=\"grype\",run_id=\"{summary.RunId}\"}} {summary.LatencyMetrics.GrypeP95Ms:F2}",
|
|
$"stellaops_parity_latency_p95_ms{{scanner=\"trivy\",run_id=\"{summary.RunId}\"}} {summary.LatencyMetrics.TrivyP95Ms:F2}",
|
|
"",
|
|
"# HELP stellaops_parity_error_scenarios_passed Number of error scenarios passed.",
|
|
"# TYPE stellaops_parity_error_scenarios_passed gauge",
|
|
$"stellaops_parity_error_scenarios_passed{{run_id=\"{summary.RunId}\"}} {summary.ErrorMetrics.ScenariosPassed}",
|
|
"",
|
|
"# HELP stellaops_parity_error_scenarios_total Total number of error scenarios.",
|
|
"# TYPE stellaops_parity_error_scenarios_total gauge",
|
|
$"stellaops_parity_error_scenarios_total{{run_id=\"{summary.RunId}\"}} {summary.ErrorMetrics.ScenariosTotal}"
|
|
};
|
|
|
|
return string.Join("\n", lines);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Exports results to InfluxDB line protocol format.
|
|
/// </summary>
|
|
public string ExportInfluxLineProtocol(ParityRunSummary summary)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(summary);
|
|
|
|
var timestamp = new DateTimeOffset(summary.Timestamp).ToUnixTimeMilliseconds() * 1_000_000; // nanoseconds
|
|
var lines = new List<string>
|
|
{
|
|
$"parity_sbom,run_id={summary.RunId} completeness_ratio={summary.SbomMetrics.PackageCompletenessRatio:F4},matched_count={summary.SbomMetrics.MatchedPackageCount}i {timestamp}",
|
|
$"parity_vuln,run_id={summary.RunId} recall={summary.VulnMetrics.Recall:F4},precision={summary.VulnMetrics.Precision:F4},f1={summary.VulnMetrics.F1Score:F4} {timestamp}",
|
|
$"parity_latency,run_id={summary.RunId},scanner=stellaops p50={summary.LatencyMetrics.StellaOpsP50Ms:F2},p95={summary.LatencyMetrics.StellaOpsP95Ms:F2},p99={summary.LatencyMetrics.StellaOpsP99Ms:F2} {timestamp}",
|
|
$"parity_latency,run_id={summary.RunId},scanner=grype p50={summary.LatencyMetrics.GrypeP50Ms:F2},p95={summary.LatencyMetrics.GrypeP95Ms:F2},p99={summary.LatencyMetrics.GrypeP99Ms:F2} {timestamp}",
|
|
$"parity_latency,run_id={summary.RunId},scanner=trivy p50={summary.LatencyMetrics.TrivyP50Ms:F2},p95={summary.LatencyMetrics.TrivyP95Ms:F2},p99={summary.LatencyMetrics.TrivyP99Ms:F2} {timestamp}",
|
|
$"parity_errors,run_id={summary.RunId} passed={summary.ErrorMetrics.ScenariosPassed}i,total={summary.ErrorMetrics.ScenariosTotal}i {timestamp}"
|
|
};
|
|
|
|
return string.Join("\n", lines);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Summary of a single parity test run.
|
|
/// </summary>
|
|
public sealed record ParityRunSummary
|
|
{
|
|
public required string RunId { get; init; }
|
|
public required DateTime Timestamp { get; init; }
|
|
public required string StellaOpsVersion { get; init; }
|
|
public required CompetitorVersions CompetitorVersions { get; init; }
|
|
public required SbomParityMetrics SbomMetrics { get; init; }
|
|
public required VulnParityMetrics VulnMetrics { get; init; }
|
|
public required LatencyParityMetrics LatencyMetrics { get; init; }
|
|
public required ErrorParityMetrics ErrorMetrics { get; init; }
|
|
public required IReadOnlyList<string> FixturesUsed { get; init; }
|
|
public string? Notes { get; init; }
|
|
}
|
|
|
|
public sealed record CompetitorVersions
|
|
{
|
|
public required string SyftVersion { get; init; }
|
|
public required string GrypeVersion { get; init; }
|
|
public required string TrivyVersion { get; init; }
|
|
}
|
|
|
|
public sealed record SbomParityMetrics
|
|
{
|
|
public required int StellaOpsPackageCount { get; init; }
|
|
public required int SyftPackageCount { get; init; }
|
|
public required int MatchedPackageCount { get; init; }
|
|
public required double PackageCompletenessRatio { get; init; }
|
|
public required double PurlCompletenessRatio { get; init; }
|
|
public required double LicenseDetectionRatio { get; init; }
|
|
public required double CpeDetectionRatio { get; init; }
|
|
}
|
|
|
|
public sealed record VulnParityMetrics
|
|
{
|
|
public required int StellaOpsCveCount { get; init; }
|
|
public required int GrypeCveCount { get; init; }
|
|
public required int TrivyCveCount { get; init; }
|
|
public required int UnionCveCount { get; init; }
|
|
public required int IntersectionCveCount { get; init; }
|
|
public required double Recall { get; init; }
|
|
public required double Precision { get; init; }
|
|
public required double F1Score { get; init; }
|
|
public required SeverityBreakdown StellaOpsSeverity { get; init; }
|
|
public required SeverityBreakdown GrypeSeverity { get; init; }
|
|
public required SeverityBreakdown TrivySeverity { get; init; }
|
|
}
|
|
|
|
public sealed record SeverityBreakdown
|
|
{
|
|
public int Critical { get; init; }
|
|
public int High { get; init; }
|
|
public int Medium { get; init; }
|
|
public int Low { get; init; }
|
|
public int Unknown { get; init; }
|
|
}
|
|
|
|
public sealed record LatencyParityMetrics
|
|
{
|
|
public required double StellaOpsP50Ms { get; init; }
|
|
public required double StellaOpsP95Ms { get; init; }
|
|
public required double StellaOpsP99Ms { get; init; }
|
|
public required double GrypeP50Ms { get; init; }
|
|
public required double GrypeP95Ms { get; init; }
|
|
public required double GrypeP99Ms { get; init; }
|
|
public required double TrivyP50Ms { get; init; }
|
|
public required double TrivyP95Ms { get; init; }
|
|
public required double TrivyP99Ms { get; init; }
|
|
public required double StellaOpsVsGrypeRatio { get; init; }
|
|
public required double StellaOpsVsTrivyRatio { get; init; }
|
|
}
|
|
|
|
public sealed record ErrorParityMetrics
|
|
{
|
|
public required int ScenariosPassed { get; init; }
|
|
public required int ScenariosTotal { get; init; }
|
|
public required IReadOnlyList<ErrorScenarioResult> ScenarioResults { get; init; }
|
|
}
|
|
|
|
public sealed record ErrorScenarioResult
|
|
{
|
|
public required string ScenarioName { get; init; }
|
|
public required bool StellaOpsPassed { get; init; }
|
|
public required bool GrypePassed { get; init; }
|
|
public required bool TrivyPassed { get; init; }
|
|
public required bool MatchesExpected { get; init; }
|
|
public string? Notes { get; init; }
|
|
}
|