// ----------------------------------------------------------------------------- // ParityHarness.cs // Sprint: SPRINT_5100_0008_0001_competitor_parity // Task: PARITY-5100-003 - Implement parity harness // Description: Harness for running StellaOps and competitors on same fixtures // ----------------------------------------------------------------------------- using System.Diagnostics; using System.Text.Json; using CliWrap; using CliWrap.Buffered; namespace StellaOps.Parity.Tests; /// /// Parity test harness that runs multiple scanners on the same container image /// and collects their outputs for comparison. /// public sealed class ParityHarness : IAsyncDisposable { private readonly string _workDir; private readonly Dictionary _toolVersions = new(); /// /// Pinned tool versions for reproducible testing. /// public static class PinnedVersions { public const string Syft = "1.9.0"; public const string Grype = "0.79.3"; public const string Trivy = "0.54.1"; } public ParityHarness(string? workDir = null) { _workDir = workDir ?? Path.Combine(Path.GetTempPath(), $"parity-{Guid.NewGuid():N}"); Directory.CreateDirectory(_workDir); } /// /// Runs all configured scanners on the specified image and returns collected results. /// public async Task RunAllAsync( ParityImageFixture fixture, CancellationToken cancellationToken = default) { var result = new ParityRunResult { Fixture = fixture, StartedAtUtc = DateTimeOffset.UtcNow }; // Run each scanner in parallel var tasks = new List> { RunSyftAsync(fixture.Image, cancellationToken), RunGrypeAsync(fixture.Image, cancellationToken), RunTrivyAsync(fixture.Image, cancellationToken) }; try { var outputs = await Task.WhenAll(tasks); result.SyftOutput = outputs[0]; result.GrypeOutput = outputs[1]; result.TrivyOutput = outputs[2]; } catch (Exception ex) { result.Error = ex.Message; } result.CompletedAtUtc = DateTimeOffset.UtcNow; return result; } /// /// Runs Syft SBOM generator on the specified image. /// public async Task RunSyftAsync( string image, CancellationToken cancellationToken = default) { var output = new ScannerOutput { ToolName = "syft", ToolVersion = PinnedVersions.Syft, Image = image, StartedAtUtc = DateTimeOffset.UtcNow }; var outputPath = Path.Combine(_workDir, $"syft-{Guid.NewGuid():N}.json"); try { var sw = Stopwatch.StartNew(); // syft -o spdx-json= var result = await Cli.Wrap("syft") .WithArguments([$"{image}", "-o", $"spdx-json={outputPath}"]) .WithValidation(CommandResultValidation.None) .ExecuteBufferedAsync(cancellationToken); sw.Stop(); output.DurationMs = sw.ElapsedMilliseconds; output.ExitCode = result.ExitCode; output.Stderr = result.StandardError; if (result.ExitCode == 0 && File.Exists(outputPath)) { output.RawOutput = await File.ReadAllTextAsync(outputPath, cancellationToken); output.SbomJson = JsonDocument.Parse(output.RawOutput); output.Success = true; } else { output.Error = result.StandardError; } } catch (Exception ex) { output.Error = ex.Message; } output.CompletedAtUtc = DateTimeOffset.UtcNow; return output; } /// /// Runs Grype vulnerability scanner on the specified image. /// public async Task RunGrypeAsync( string image, CancellationToken cancellationToken = default) { var output = new ScannerOutput { ToolName = "grype", ToolVersion = PinnedVersions.Grype, Image = image, StartedAtUtc = DateTimeOffset.UtcNow }; var outputPath = Path.Combine(_workDir, $"grype-{Guid.NewGuid():N}.json"); try { var sw = Stopwatch.StartNew(); // grype -o json --file var result = await Cli.Wrap("grype") .WithArguments([$"{image}", "-o", "json", "--file", outputPath]) .WithValidation(CommandResultValidation.None) .ExecuteBufferedAsync(cancellationToken); sw.Stop(); output.DurationMs = sw.ElapsedMilliseconds; output.ExitCode = result.ExitCode; output.Stderr = result.StandardError; if (result.ExitCode == 0 && File.Exists(outputPath)) { output.RawOutput = await File.ReadAllTextAsync(outputPath, cancellationToken); output.FindingsJson = JsonDocument.Parse(output.RawOutput); output.Success = true; } else { output.Error = result.StandardError; } } catch (Exception ex) { output.Error = ex.Message; } output.CompletedAtUtc = DateTimeOffset.UtcNow; return output; } /// /// Runs Trivy vulnerability scanner on the specified image. /// public async Task RunTrivyAsync( string image, CancellationToken cancellationToken = default) { var output = new ScannerOutput { ToolName = "trivy", ToolVersion = PinnedVersions.Trivy, Image = image, StartedAtUtc = DateTimeOffset.UtcNow }; var outputPath = Path.Combine(_workDir, $"trivy-{Guid.NewGuid():N}.json"); try { var sw = Stopwatch.StartNew(); // trivy image -f json -o var result = await Cli.Wrap("trivy") .WithArguments(["image", image, "-f", "json", "-o", outputPath]) .WithValidation(CommandResultValidation.None) .ExecuteBufferedAsync(cancellationToken); sw.Stop(); output.DurationMs = sw.ElapsedMilliseconds; output.ExitCode = result.ExitCode; output.Stderr = result.StandardError; if (result.ExitCode == 0 && File.Exists(outputPath)) { output.RawOutput = await File.ReadAllTextAsync(outputPath, cancellationToken); output.FindingsJson = JsonDocument.Parse(output.RawOutput); output.Success = true; } else { output.Error = result.StandardError; } } catch (Exception ex) { output.Error = ex.Message; } output.CompletedAtUtc = DateTimeOffset.UtcNow; return output; } /// /// Checks if required tools are available on the system. /// public async Task CheckToolsAsync(CancellationToken cancellationToken = default) { var availability = new ToolAvailability(); availability.SyftAvailable = await CheckToolAsync("syft", "--version", cancellationToken); availability.GrypeAvailable = await CheckToolAsync("grype", "--version", cancellationToken); availability.TrivyAvailable = await CheckToolAsync("trivy", "--version", cancellationToken); return availability; } private static async Task CheckToolAsync(string tool, string versionArg, CancellationToken cancellationToken) { try { var result = await Cli.Wrap(tool) .WithArguments([versionArg]) .WithValidation(CommandResultValidation.None) .ExecuteBufferedAsync(cancellationToken); return result.ExitCode == 0; } catch { return false; } } public ValueTask DisposeAsync() { if (Directory.Exists(_workDir)) { try { Directory.Delete(_workDir, recursive: true); } catch { // Ignore cleanup errors } } return ValueTask.CompletedTask; } } /// /// Result of running all scanners on a fixture. /// public sealed class ParityRunResult { public required ParityImageFixture Fixture { get; init; } public DateTimeOffset StartedAtUtc { get; set; } public DateTimeOffset CompletedAtUtc { get; set; } public ScannerOutput? SyftOutput { get; set; } public ScannerOutput? GrypeOutput { get; set; } public ScannerOutput? TrivyOutput { get; set; } public string? Error { get; set; } public TimeSpan Duration => CompletedAtUtc - StartedAtUtc; } /// /// Output from a single scanner run. /// public sealed class ScannerOutput { public required string ToolName { get; init; } public required string ToolVersion { get; init; } public required string Image { get; init; } public DateTimeOffset StartedAtUtc { get; set; } public DateTimeOffset CompletedAtUtc { get; set; } public long DurationMs { get; set; } public int ExitCode { get; set; } public string? RawOutput { get; set; } public string? Stderr { get; set; } public string? Error { get; set; } public bool Success { get; set; } /// SBOM JSON document (for Syft). public JsonDocument? SbomJson { get; set; } /// Vulnerability findings JSON document (for Grype/Trivy). public JsonDocument? FindingsJson { get; set; } } /// /// Tool availability check result. /// public sealed class ToolAvailability { public bool SyftAvailable { get; set; } public bool GrypeAvailable { get; set; } public bool TrivyAvailable { get; set; } public bool AllAvailable => SyftAvailable && GrypeAvailable && TrivyAvailable; public bool AnyAvailable => SyftAvailable || GrypeAvailable || TrivyAvailable; } /// /// Cached tool version information. /// public sealed class ToolVersion { public required string ToolName { get; init; } public required string Version { get; init; } public DateTimeOffset DetectedAtUtc { get; init; } }