5100* tests strengthtenen work
This commit is contained in:
StellaOps Bot
341
tests/parity/StellaOps.Parity.Tests/ParityHarness.cs
Normal file
341
tests/parity/StellaOps.Parity.Tests/ParityHarness.cs
Normal file
@@ -0,0 +1,341 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// ParityHarness.cs
|
||||
// Sprint: SPRINT_5100_0008_0001_competitor_parity
|
||||
// Task: PARITY-5100-003 - Implement parity harness
|
||||
// Description: Harness for running StellaOps and competitors on same fixtures
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Diagnostics;
|
||||
using System.Text.Json;
|
||||
using CliWrap;
|
||||
using CliWrap.Buffered;
|
||||
|
||||
namespace StellaOps.Parity.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Parity test harness that runs multiple scanners on the same container image
|
||||
/// and collects their outputs for comparison.
|
||||
/// </summary>
|
||||
public sealed class ParityHarness : IAsyncDisposable
|
||||
{
|
||||
private readonly string _workDir;
|
||||
private readonly Dictionary<string, ToolVersion> _toolVersions = new();
|
||||
|
||||
/// <summary>
|
||||
/// Pinned tool versions for reproducible testing.
|
||||
/// </summary>
|
||||
public static class PinnedVersions
|
||||
{
|
||||
public const string Syft = "1.9.0";
|
||||
public const string Grype = "0.79.3";
|
||||
public const string Trivy = "0.54.1";
|
||||
}
|
||||
|
||||
public ParityHarness(string? workDir = null)
|
||||
{
|
||||
_workDir = workDir ?? Path.Combine(Path.GetTempPath(), $"parity-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(_workDir);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs all configured scanners on the specified image and returns collected results.
|
||||
/// </summary>
|
||||
public async Task<ParityRunResult> RunAllAsync(
|
||||
ParityImageFixture fixture,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var result = new ParityRunResult
|
||||
{
|
||||
Fixture = fixture,
|
||||
StartedAtUtc = DateTimeOffset.UtcNow
|
||||
};
|
||||
|
||||
// Run each scanner in parallel
|
||||
var tasks = new List<Task<ScannerOutput>>
|
||||
{
|
||||
RunSyftAsync(fixture.Image, cancellationToken),
|
||||
RunGrypeAsync(fixture.Image, cancellationToken),
|
||||
RunTrivyAsync(fixture.Image, cancellationToken)
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
var outputs = await Task.WhenAll(tasks);
|
||||
result.SyftOutput = outputs[0];
|
||||
result.GrypeOutput = outputs[1];
|
||||
result.TrivyOutput = outputs[2];
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
result.Error = ex.Message;
|
||||
}
|
||||
|
||||
result.CompletedAtUtc = DateTimeOffset.UtcNow;
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs Syft SBOM generator on the specified image.
|
||||
/// </summary>
|
||||
public async Task<ScannerOutput> RunSyftAsync(
|
||||
string image,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var output = new ScannerOutput
|
||||
{
|
||||
ToolName = "syft",
|
||||
ToolVersion = PinnedVersions.Syft,
|
||||
Image = image,
|
||||
StartedAtUtc = DateTimeOffset.UtcNow
|
||||
};
|
||||
|
||||
var outputPath = Path.Combine(_workDir, $"syft-{Guid.NewGuid():N}.json");
|
||||
|
||||
try
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
|
||||
// syft <image> -o spdx-json=<output>
|
||||
var result = await Cli.Wrap("syft")
|
||||
.WithArguments([$"{image}", "-o", $"spdx-json={outputPath}"])
|
||||
.WithValidation(CommandResultValidation.None)
|
||||
.ExecuteBufferedAsync(cancellationToken);
|
||||
|
||||
sw.Stop();
|
||||
output.DurationMs = sw.ElapsedMilliseconds;
|
||||
output.ExitCode = result.ExitCode;
|
||||
output.Stderr = result.StandardError;
|
||||
|
||||
if (result.ExitCode == 0 && File.Exists(outputPath))
|
||||
{
|
||||
output.RawOutput = await File.ReadAllTextAsync(outputPath, cancellationToken);
|
||||
output.SbomJson = JsonDocument.Parse(output.RawOutput);
|
||||
output.Success = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
output.Error = result.StandardError;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
output.Error = ex.Message;
|
||||
}
|
||||
|
||||
output.CompletedAtUtc = DateTimeOffset.UtcNow;
|
||||
return output;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs Grype vulnerability scanner on the specified image.
|
||||
/// </summary>
|
||||
public async Task<ScannerOutput> RunGrypeAsync(
|
||||
string image,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var output = new ScannerOutput
|
||||
{
|
||||
ToolName = "grype",
|
||||
ToolVersion = PinnedVersions.Grype,
|
||||
Image = image,
|
||||
StartedAtUtc = DateTimeOffset.UtcNow
|
||||
};
|
||||
|
||||
var outputPath = Path.Combine(_workDir, $"grype-{Guid.NewGuid():N}.json");
|
||||
|
||||
try
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
|
||||
// grype <image> -o json --file <output>
|
||||
var result = await Cli.Wrap("grype")
|
||||
.WithArguments([$"{image}", "-o", "json", "--file", outputPath])
|
||||
.WithValidation(CommandResultValidation.None)
|
||||
.ExecuteBufferedAsync(cancellationToken);
|
||||
|
||||
sw.Stop();
|
||||
output.DurationMs = sw.ElapsedMilliseconds;
|
||||
output.ExitCode = result.ExitCode;
|
||||
output.Stderr = result.StandardError;
|
||||
|
||||
if (result.ExitCode == 0 && File.Exists(outputPath))
|
||||
{
|
||||
output.RawOutput = await File.ReadAllTextAsync(outputPath, cancellationToken);
|
||||
output.FindingsJson = JsonDocument.Parse(output.RawOutput);
|
||||
output.Success = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
output.Error = result.StandardError;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
output.Error = ex.Message;
|
||||
}
|
||||
|
||||
output.CompletedAtUtc = DateTimeOffset.UtcNow;
|
||||
return output;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs Trivy vulnerability scanner on the specified image.
|
||||
/// </summary>
|
||||
public async Task<ScannerOutput> RunTrivyAsync(
|
||||
string image,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var output = new ScannerOutput
|
||||
{
|
||||
ToolName = "trivy",
|
||||
ToolVersion = PinnedVersions.Trivy,
|
||||
Image = image,
|
||||
StartedAtUtc = DateTimeOffset.UtcNow
|
||||
};
|
||||
|
||||
var outputPath = Path.Combine(_workDir, $"trivy-{Guid.NewGuid():N}.json");
|
||||
|
||||
try
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
|
||||
// trivy image <image> -f json -o <output>
|
||||
var result = await Cli.Wrap("trivy")
|
||||
.WithArguments(["image", image, "-f", "json", "-o", outputPath])
|
||||
.WithValidation(CommandResultValidation.None)
|
||||
.ExecuteBufferedAsync(cancellationToken);
|
||||
|
||||
sw.Stop();
|
||||
output.DurationMs = sw.ElapsedMilliseconds;
|
||||
output.ExitCode = result.ExitCode;
|
||||
output.Stderr = result.StandardError;
|
||||
|
||||
if (result.ExitCode == 0 && File.Exists(outputPath))
|
||||
{
|
||||
output.RawOutput = await File.ReadAllTextAsync(outputPath, cancellationToken);
|
||||
output.FindingsJson = JsonDocument.Parse(output.RawOutput);
|
||||
output.Success = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
output.Error = result.StandardError;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
output.Error = ex.Message;
|
||||
}
|
||||
|
||||
output.CompletedAtUtc = DateTimeOffset.UtcNow;
|
||||
return output;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if required tools are available on the system.
|
||||
/// </summary>
|
||||
public async Task<ToolAvailability> CheckToolsAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var availability = new ToolAvailability();
|
||||
|
||||
availability.SyftAvailable = await CheckToolAsync("syft", "--version", cancellationToken);
|
||||
availability.GrypeAvailable = await CheckToolAsync("grype", "--version", cancellationToken);
|
||||
availability.TrivyAvailable = await CheckToolAsync("trivy", "--version", cancellationToken);
|
||||
|
||||
return availability;
|
||||
}
|
||||
|
||||
private static async Task<bool> CheckToolAsync(string tool, string versionArg, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var result = await Cli.Wrap(tool)
|
||||
.WithArguments([versionArg])
|
||||
.WithValidation(CommandResultValidation.None)
|
||||
.ExecuteBufferedAsync(cancellationToken);
|
||||
return result.ExitCode == 0;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
if (Directory.Exists(_workDir))
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.Delete(_workDir, recursive: true);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
}
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of running all scanners on a fixture.
|
||||
/// </summary>
|
||||
public sealed class ParityRunResult
|
||||
{
|
||||
public required ParityImageFixture Fixture { get; init; }
|
||||
public DateTimeOffset StartedAtUtc { get; set; }
|
||||
public DateTimeOffset CompletedAtUtc { get; set; }
|
||||
public ScannerOutput? SyftOutput { get; set; }
|
||||
public ScannerOutput? GrypeOutput { get; set; }
|
||||
public ScannerOutput? TrivyOutput { get; set; }
|
||||
public string? Error { get; set; }
|
||||
|
||||
public TimeSpan Duration => CompletedAtUtc - StartedAtUtc;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Output from a single scanner run.
|
||||
/// </summary>
|
||||
public sealed class ScannerOutput
|
||||
{
|
||||
public required string ToolName { get; init; }
|
||||
public required string ToolVersion { get; init; }
|
||||
public required string Image { get; init; }
|
||||
public DateTimeOffset StartedAtUtc { get; set; }
|
||||
public DateTimeOffset CompletedAtUtc { get; set; }
|
||||
public long DurationMs { get; set; }
|
||||
public int ExitCode { get; set; }
|
||||
public string? RawOutput { get; set; }
|
||||
public string? Stderr { get; set; }
|
||||
public string? Error { get; set; }
|
||||
public bool Success { get; set; }
|
||||
|
||||
/// <summary>SBOM JSON document (for Syft).</summary>
|
||||
public JsonDocument? SbomJson { get; set; }
|
||||
|
||||
/// <summary>Vulnerability findings JSON document (for Grype/Trivy).</summary>
|
||||
public JsonDocument? FindingsJson { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tool availability check result.
|
||||
/// </summary>
|
||||
public sealed class ToolAvailability
|
||||
{
|
||||
public bool SyftAvailable { get; set; }
|
||||
public bool GrypeAvailable { get; set; }
|
||||
public bool TrivyAvailable { get; set; }
|
||||
|
||||
public bool AllAvailable => SyftAvailable && GrypeAvailable && TrivyAvailable;
|
||||
public bool AnyAvailable => SyftAvailable || GrypeAvailable || TrivyAvailable;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cached tool version information.
|
||||
/// </summary>
|
||||
public sealed class ToolVersion
|
||||
{
|
||||
public required string ToolName { get; init; }
|
||||
public required string Version { get; init; }
|
||||
public DateTimeOffset DetectedAtUtc { get; init; }
|
||||
}
|
||||
Reference in New Issue
Block a user