342 lines
11 KiB
C#
342 lines
11 KiB
C#
// -----------------------------------------------------------------------------
|
|
// ParityHarness.cs
|
|
// Sprint: SPRINT_5100_0008_0001_competitor_parity
|
|
// Task: PARITY-5100-003 - Implement parity harness
|
|
// Description: Harness for running StellaOps and competitors on same fixtures
|
|
// -----------------------------------------------------------------------------
|
|
|
|
using System.Diagnostics;
|
|
using System.Text.Json;
|
|
using CliWrap;
|
|
using CliWrap.Buffered;
|
|
|
|
namespace StellaOps.Parity.Tests;
|
|
|
|
/// <summary>
|
|
/// Parity test harness that runs multiple scanners on the same container image
|
|
/// and collects their outputs for comparison.
|
|
/// </summary>
|
|
public sealed class ParityHarness : IAsyncDisposable
|
|
{
|
|
private readonly string _workDir;
|
|
private readonly Dictionary<string, ToolVersion> _toolVersions = new();
|
|
|
|
/// <summary>
|
|
/// Pinned tool versions for reproducible testing.
|
|
/// </summary>
|
|
public static class PinnedVersions
|
|
{
|
|
public const string Syft = "1.9.0";
|
|
public const string Grype = "0.79.3";
|
|
public const string Trivy = "0.54.1";
|
|
}
|
|
|
|
public ParityHarness(string? workDir = null)
|
|
{
|
|
_workDir = workDir ?? Path.Combine(Path.GetTempPath(), $"parity-{Guid.NewGuid():N}");
|
|
Directory.CreateDirectory(_workDir);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Runs all configured scanners on the specified image and returns collected results.
|
|
/// </summary>
|
|
public async Task<ParityRunResult> RunAllAsync(
|
|
ParityImageFixture fixture,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var result = new ParityRunResult
|
|
{
|
|
Fixture = fixture,
|
|
StartedAtUtc = DateTimeOffset.UtcNow
|
|
};
|
|
|
|
// Run each scanner in parallel
|
|
var tasks = new List<Task<ScannerOutput>>
|
|
{
|
|
RunSyftAsync(fixture.Image, cancellationToken),
|
|
RunGrypeAsync(fixture.Image, cancellationToken),
|
|
RunTrivyAsync(fixture.Image, cancellationToken)
|
|
};
|
|
|
|
try
|
|
{
|
|
var outputs = await Task.WhenAll(tasks);
|
|
result.SyftOutput = outputs[0];
|
|
result.GrypeOutput = outputs[1];
|
|
result.TrivyOutput = outputs[2];
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
result.Error = ex.Message;
|
|
}
|
|
|
|
result.CompletedAtUtc = DateTimeOffset.UtcNow;
|
|
return result;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Runs Syft SBOM generator on the specified image.
|
|
/// </summary>
|
|
public async Task<ScannerOutput> RunSyftAsync(
|
|
string image,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var output = new ScannerOutput
|
|
{
|
|
ToolName = "syft",
|
|
ToolVersion = PinnedVersions.Syft,
|
|
Image = image,
|
|
StartedAtUtc = DateTimeOffset.UtcNow
|
|
};
|
|
|
|
var outputPath = Path.Combine(_workDir, $"syft-{Guid.NewGuid():N}.json");
|
|
|
|
try
|
|
{
|
|
var sw = Stopwatch.StartNew();
|
|
|
|
// syft <image> -o spdx-json=<output>
|
|
var result = await Cli.Wrap("syft")
|
|
.WithArguments([$"{image}", "-o", $"spdx-json={outputPath}"])
|
|
.WithValidation(CommandResultValidation.None)
|
|
.ExecuteBufferedAsync(cancellationToken);
|
|
|
|
sw.Stop();
|
|
output.DurationMs = sw.ElapsedMilliseconds;
|
|
output.ExitCode = result.ExitCode;
|
|
output.Stderr = result.StandardError;
|
|
|
|
if (result.ExitCode == 0 && File.Exists(outputPath))
|
|
{
|
|
output.RawOutput = await File.ReadAllTextAsync(outputPath, cancellationToken);
|
|
output.SbomJson = JsonDocument.Parse(output.RawOutput);
|
|
output.Success = true;
|
|
}
|
|
else
|
|
{
|
|
output.Error = result.StandardError;
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
output.Error = ex.Message;
|
|
}
|
|
|
|
output.CompletedAtUtc = DateTimeOffset.UtcNow;
|
|
return output;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Runs Grype vulnerability scanner on the specified image.
|
|
/// </summary>
|
|
public async Task<ScannerOutput> RunGrypeAsync(
|
|
string image,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var output = new ScannerOutput
|
|
{
|
|
ToolName = "grype",
|
|
ToolVersion = PinnedVersions.Grype,
|
|
Image = image,
|
|
StartedAtUtc = DateTimeOffset.UtcNow
|
|
};
|
|
|
|
var outputPath = Path.Combine(_workDir, $"grype-{Guid.NewGuid():N}.json");
|
|
|
|
try
|
|
{
|
|
var sw = Stopwatch.StartNew();
|
|
|
|
// grype <image> -o json --file <output>
|
|
var result = await Cli.Wrap("grype")
|
|
.WithArguments([$"{image}", "-o", "json", "--file", outputPath])
|
|
.WithValidation(CommandResultValidation.None)
|
|
.ExecuteBufferedAsync(cancellationToken);
|
|
|
|
sw.Stop();
|
|
output.DurationMs = sw.ElapsedMilliseconds;
|
|
output.ExitCode = result.ExitCode;
|
|
output.Stderr = result.StandardError;
|
|
|
|
if (result.ExitCode == 0 && File.Exists(outputPath))
|
|
{
|
|
output.RawOutput = await File.ReadAllTextAsync(outputPath, cancellationToken);
|
|
output.FindingsJson = JsonDocument.Parse(output.RawOutput);
|
|
output.Success = true;
|
|
}
|
|
else
|
|
{
|
|
output.Error = result.StandardError;
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
output.Error = ex.Message;
|
|
}
|
|
|
|
output.CompletedAtUtc = DateTimeOffset.UtcNow;
|
|
return output;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Runs Trivy vulnerability scanner on the specified image.
|
|
/// </summary>
|
|
public async Task<ScannerOutput> RunTrivyAsync(
|
|
string image,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var output = new ScannerOutput
|
|
{
|
|
ToolName = "trivy",
|
|
ToolVersion = PinnedVersions.Trivy,
|
|
Image = image,
|
|
StartedAtUtc = DateTimeOffset.UtcNow
|
|
};
|
|
|
|
var outputPath = Path.Combine(_workDir, $"trivy-{Guid.NewGuid():N}.json");
|
|
|
|
try
|
|
{
|
|
var sw = Stopwatch.StartNew();
|
|
|
|
// trivy image <image> -f json -o <output>
|
|
var result = await Cli.Wrap("trivy")
|
|
.WithArguments(["image", image, "-f", "json", "-o", outputPath])
|
|
.WithValidation(CommandResultValidation.None)
|
|
.ExecuteBufferedAsync(cancellationToken);
|
|
|
|
sw.Stop();
|
|
output.DurationMs = sw.ElapsedMilliseconds;
|
|
output.ExitCode = result.ExitCode;
|
|
output.Stderr = result.StandardError;
|
|
|
|
if (result.ExitCode == 0 && File.Exists(outputPath))
|
|
{
|
|
output.RawOutput = await File.ReadAllTextAsync(outputPath, cancellationToken);
|
|
output.FindingsJson = JsonDocument.Parse(output.RawOutput);
|
|
output.Success = true;
|
|
}
|
|
else
|
|
{
|
|
output.Error = result.StandardError;
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
output.Error = ex.Message;
|
|
}
|
|
|
|
output.CompletedAtUtc = DateTimeOffset.UtcNow;
|
|
return output;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Checks if required tools are available on the system.
|
|
/// </summary>
|
|
public async Task<ToolAvailability> CheckToolsAsync(CancellationToken cancellationToken = default)
|
|
{
|
|
var availability = new ToolAvailability();
|
|
|
|
availability.SyftAvailable = await CheckToolAsync("syft", "--version", cancellationToken);
|
|
availability.GrypeAvailable = await CheckToolAsync("grype", "--version", cancellationToken);
|
|
availability.TrivyAvailable = await CheckToolAsync("trivy", "--version", cancellationToken);
|
|
|
|
return availability;
|
|
}
|
|
|
|
private static async Task<bool> CheckToolAsync(string tool, string versionArg, CancellationToken cancellationToken)
|
|
{
|
|
try
|
|
{
|
|
var result = await Cli.Wrap(tool)
|
|
.WithArguments([versionArg])
|
|
.WithValidation(CommandResultValidation.None)
|
|
.ExecuteBufferedAsync(cancellationToken);
|
|
return result.ExitCode == 0;
|
|
}
|
|
catch
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
public ValueTask DisposeAsync()
|
|
{
|
|
if (Directory.Exists(_workDir))
|
|
{
|
|
try
|
|
{
|
|
Directory.Delete(_workDir, recursive: true);
|
|
}
|
|
catch
|
|
{
|
|
// Ignore cleanup errors
|
|
}
|
|
}
|
|
return ValueTask.CompletedTask;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Result of running all scanners on a fixture.
|
|
/// </summary>
|
|
public sealed class ParityRunResult
|
|
{
|
|
public required ParityImageFixture Fixture { get; init; }
|
|
public DateTimeOffset StartedAtUtc { get; set; }
|
|
public DateTimeOffset CompletedAtUtc { get; set; }
|
|
public ScannerOutput? SyftOutput { get; set; }
|
|
public ScannerOutput? GrypeOutput { get; set; }
|
|
public ScannerOutput? TrivyOutput { get; set; }
|
|
public string? Error { get; set; }
|
|
|
|
public TimeSpan Duration => CompletedAtUtc - StartedAtUtc;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Output from a single scanner run.
|
|
/// </summary>
|
|
public sealed class ScannerOutput
|
|
{
|
|
public required string ToolName { get; init; }
|
|
public required string ToolVersion { get; init; }
|
|
public required string Image { get; init; }
|
|
public DateTimeOffset StartedAtUtc { get; set; }
|
|
public DateTimeOffset CompletedAtUtc { get; set; }
|
|
public long DurationMs { get; set; }
|
|
public int ExitCode { get; set; }
|
|
public string? RawOutput { get; set; }
|
|
public string? Stderr { get; set; }
|
|
public string? Error { get; set; }
|
|
public bool Success { get; set; }
|
|
|
|
/// <summary>SBOM JSON document (for Syft).</summary>
|
|
public JsonDocument? SbomJson { get; set; }
|
|
|
|
/// <summary>Vulnerability findings JSON document (for Grype/Trivy).</summary>
|
|
public JsonDocument? FindingsJson { get; set; }
|
|
}
|
|
|
|
/// <summary>
|
|
/// Tool availability check result.
|
|
/// </summary>
|
|
public sealed class ToolAvailability
|
|
{
|
|
public bool SyftAvailable { get; set; }
|
|
public bool GrypeAvailable { get; set; }
|
|
public bool TrivyAvailable { get; set; }
|
|
|
|
public bool AllAvailable => SyftAvailable && GrypeAvailable && TrivyAvailable;
|
|
public bool AnyAvailable => SyftAvailable || GrypeAvailable || TrivyAvailable;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Cached tool version information.
|
|
/// </summary>
|
|
public sealed class ToolVersion
|
|
{
|
|
public required string ToolName { get; init; }
|
|
public required string Version { get; init; }
|
|
public DateTimeOffset DetectedAtUtc { get; init; }
|
|
}
|