458 lines
19 KiB
C#
458 lines
19 KiB
C#
// -----------------------------------------------------------------------------
|
|
// E2EReproducibilityTests.cs
|
|
// Sprint: SPRINT_8200_0001_0004_e2e_reproducibility_test
|
|
// Tasks: E2E-8200-011 to E2E-8200-014 - Reproducibility Tests
|
|
// Description: End-to-end tests verifying full pipeline reproducibility.
|
|
// Validates: identical verdict hash, identical manifest, frozen timestamps,
|
|
// parallel execution produces identical results.
|
|
// -----------------------------------------------------------------------------
|
|
|
|
using FluentAssertions;
|
|
using Xunit;
|
|
|
|
namespace StellaOps.Integration.E2E;
|
|
|
|
/// <summary>
|
|
/// End-to-end reproducibility tests for the full security scanning pipeline.
|
|
/// Verifies that identical inputs always produce identical outputs across:
|
|
/// - Sequential runs
|
|
/// - Parallel runs
|
|
/// - With frozen timestamps
|
|
/// </summary>
|
|
[Collection("E2EReproducibility")]
|
|
[Trait("Category", "Integration")]
|
|
[Trait("Sprint", "8200.0001.0004")]
|
|
[Trait("Feature", "E2E-Reproducibility")]
|
|
public sealed class E2EReproducibilityTests : IClassFixture<E2EReproducibilityTestFixture>, IAsyncLifetime
|
|
{
|
|
private readonly E2EReproducibilityTestFixture _fixture;
|
|
|
|
public E2EReproducibilityTests(E2EReproducibilityTestFixture fixture)
|
|
{
|
|
_fixture = fixture;
|
|
}
|
|
|
|
public async Task InitializeAsync()
|
|
{
|
|
await _fixture.InitializeAsync();
|
|
}
|
|
|
|
public Task DisposeAsync() => Task.CompletedTask;
|
|
|
|
#region E2E-8200-011: Identical Verdict Hash
|
|
|
|
[Fact(DisplayName = "Pipeline produces identical verdict hash across runs")]
|
|
public async Task FullPipeline_ProducesIdenticalVerdictHash_AcrossRuns()
|
|
{
|
|
// Arrange - Create input snapshot
|
|
var inputs = await _fixture.SnapshotInputsAsync();
|
|
|
|
// Act - Run pipeline twice with identical inputs
|
|
var result1 = await _fixture.RunFullPipelineAsync(inputs);
|
|
var result2 = await _fixture.RunFullPipelineAsync(inputs);
|
|
|
|
// Assert - Verdict IDs must match
|
|
result1.VerdictId.Should().NotBeNullOrEmpty("Verdict ID should be computed");
|
|
result2.VerdictId.Should().NotBeNullOrEmpty("Verdict ID should be computed");
|
|
result1.VerdictId.Should().Be(result2.VerdictId, "Verdict ID must be identical across runs");
|
|
|
|
// Verdict hash must match
|
|
result1.VerdictHash.Should().Be(result2.VerdictHash, "Verdict hash must be identical");
|
|
}
|
|
|
|
[Fact(DisplayName = "Pipeline produces identical verdict hash with 5 sequential runs")]
|
|
public async Task FullPipeline_ProducesIdenticalVerdictHash_With5SequentialRuns()
|
|
{
|
|
// Arrange
|
|
var inputs = await _fixture.SnapshotInputsAsync();
|
|
var results = new List<PipelineResult>();
|
|
|
|
// Act - Run pipeline 5 times sequentially
|
|
for (int i = 0; i < 5; i++)
|
|
{
|
|
results.Add(await _fixture.RunFullPipelineAsync(inputs));
|
|
}
|
|
|
|
// Assert - All verdict IDs must match
|
|
var firstVerdictId = results[0].VerdictId;
|
|
foreach (var result in results)
|
|
{
|
|
result.VerdictId.Should().Be(firstVerdictId, $"Run {results.IndexOf(result) + 1} verdict ID must match first run");
|
|
}
|
|
}
|
|
|
|
[Fact(DisplayName = "Verdict ID format is content-addressed")]
|
|
public async Task VerdictId_Format_IsContentAddressed()
|
|
{
|
|
// Arrange
|
|
var inputs = await _fixture.SnapshotInputsAsync();
|
|
|
|
// Act
|
|
var result = await _fixture.RunFullPipelineAsync(inputs);
|
|
|
|
// Assert - Verdict ID should be in content-addressed format
|
|
result.VerdictId.Should().StartWith("verdict:sha256:", "Verdict ID must use sha256 content-addressing");
|
|
result.VerdictId.Should().MatchRegex(@"^verdict:sha256:[0-9a-f]{64}$", "Verdict ID must be valid sha256 hex");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region E2E-8200-012: Identical Bundle Manifest
|
|
|
|
[Fact(DisplayName = "Pipeline produces identical bundle manifest across runs")]
|
|
public async Task FullPipeline_ProducesIdenticalBundleManifest_AcrossRuns()
|
|
{
|
|
// Arrange
|
|
var inputs = await _fixture.SnapshotInputsAsync();
|
|
|
|
// Act - Run pipeline twice
|
|
var result1 = await _fixture.RunFullPipelineAsync(inputs);
|
|
var result2 = await _fixture.RunFullPipelineAsync(inputs);
|
|
|
|
// Assert - Bundle manifests must be byte-for-byte identical
|
|
result1.BundleManifest.Should().BeEquivalentTo(result2.BundleManifest, "Bundle manifest bytes must match");
|
|
result1.BundleManifestHash.Should().Be(result2.BundleManifestHash, "Bundle manifest hash must match");
|
|
}
|
|
|
|
[Fact(DisplayName = "Bundle manifest contains all artifact hashes")]
|
|
public async Task BundleManifest_ContainsAllArtifactHashes()
|
|
{
|
|
// Arrange
|
|
var inputs = await _fixture.SnapshotInputsAsync();
|
|
|
|
// Act
|
|
var result = await _fixture.RunFullPipelineAsync(inputs);
|
|
|
|
// Assert - Parse manifest and verify structure
|
|
var manifestJson = System.Text.Encoding.UTF8.GetString(result.BundleManifest);
|
|
using var doc = System.Text.Json.JsonDocument.Parse(manifestJson);
|
|
var root = doc.RootElement;
|
|
|
|
root.TryGetProperty("version", out _).Should().BeTrue("Manifest should have version");
|
|
root.TryGetProperty("createdAt", out _).Should().BeTrue("Manifest should have createdAt");
|
|
root.TryGetProperty("artifacts", out var artifacts).Should().BeTrue("Manifest should have artifacts");
|
|
|
|
artifacts.TryGetProperty("sbom", out _).Should().BeTrue("Artifacts should include SBOM hash");
|
|
artifacts.TryGetProperty("advisory-feed", out _).Should().BeTrue("Artifacts should include advisory feed hash");
|
|
artifacts.TryGetProperty("policy-pack", out _).Should().BeTrue("Artifacts should include policy pack hash");
|
|
artifacts.TryGetProperty("envelope", out _).Should().BeTrue("Artifacts should include envelope hash");
|
|
}
|
|
|
|
[Fact(DisplayName = "Manifest comparison detects differences")]
|
|
public async Task ManifestComparer_DetectsDifferences_WhenInputsChange()
|
|
{
|
|
// Arrange - Create two different input snapshots
|
|
var inputs1 = await _fixture.SnapshotInputsAsync();
|
|
|
|
// Modify SBOM to create different input
|
|
var modifiedSbom = E2EReproducibilityTestFixture.CreateMinimalSbom();
|
|
var sbomJson = System.Text.Encoding.UTF8.GetString(modifiedSbom);
|
|
var modifiedSbomJson = sbomJson.Replace("4.17.20", "4.17.21"); // Change version
|
|
var modifiedSbomBytes = System.Text.Encoding.UTF8.GetBytes(modifiedSbomJson);
|
|
|
|
var inputs2 = new InputSnapshot
|
|
{
|
|
Sbom = modifiedSbomBytes,
|
|
SbomHash = E2EReproducibilityTestFixture.ComputeHash(modifiedSbomBytes),
|
|
AdvisoryFeed = inputs1.AdvisoryFeed,
|
|
AdvisoryFeedHash = inputs1.AdvisoryFeedHash,
|
|
PolicyPack = inputs1.PolicyPack,
|
|
PolicyPackHash = inputs1.PolicyPackHash,
|
|
VexDocument = inputs1.VexDocument,
|
|
VexDocumentHash = inputs1.VexDocumentHash,
|
|
SnapshotTimestamp = inputs1.SnapshotTimestamp
|
|
};
|
|
|
|
// Act
|
|
var result1 = await _fixture.RunFullPipelineAsync(inputs1);
|
|
var result2 = await _fixture.RunFullPipelineAsync(inputs2);
|
|
|
|
// Assert - Results should differ
|
|
var comparison = ManifestComparer.Compare(result1, result2);
|
|
comparison.IsMatch.Should().BeFalse("Different inputs should produce different outputs");
|
|
comparison.Differences.Should().NotBeEmpty("Should detect at least one difference");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region E2E-8200-013: Frozen Clock Timestamps
|
|
|
|
[Fact(DisplayName = "Pipeline produces identical timestamps with frozen clock")]
|
|
public async Task FullPipeline_ProducesIdenticalTimestamps_WithFrozenClock()
|
|
{
|
|
// Arrange
|
|
var inputs = await _fixture.SnapshotInputsAsync();
|
|
|
|
// Act - Run pipeline twice
|
|
var result1 = await _fixture.RunFullPipelineAsync(inputs);
|
|
var result2 = await _fixture.RunFullPipelineAsync(inputs);
|
|
|
|
// Assert - Execution timestamps must match (frozen clock)
|
|
result1.ExecutionTimestamp.Should().Be(_fixture.FrozenTimestamp, "Timestamp should match frozen clock");
|
|
result2.ExecutionTimestamp.Should().Be(_fixture.FrozenTimestamp, "Timestamp should match frozen clock");
|
|
result1.ExecutionTimestamp.Should().Be(result2.ExecutionTimestamp, "Timestamps must be identical");
|
|
}
|
|
|
|
[Fact(DisplayName = "Manifest createdAt matches frozen timestamp")]
|
|
public async Task BundleManifest_CreatedAt_MatchesFrozenTimestamp()
|
|
{
|
|
// Arrange
|
|
var inputs = await _fixture.SnapshotInputsAsync();
|
|
|
|
// Act
|
|
var result = await _fixture.RunFullPipelineAsync(inputs);
|
|
|
|
// Assert - Parse manifest and verify timestamp
|
|
var manifestJson = System.Text.Encoding.UTF8.GetString(result.BundleManifest);
|
|
using var doc = System.Text.Json.JsonDocument.Parse(manifestJson);
|
|
var createdAt = doc.RootElement.GetProperty("createdAt").GetDateTimeOffset();
|
|
|
|
createdAt.Should().Be(_fixture.FrozenTimestamp, "Manifest createdAt should match frozen clock");
|
|
}
|
|
|
|
[Fact(DisplayName = "Input snapshot timestamp matches frozen clock")]
|
|
public async Task InputSnapshot_Timestamp_MatchesFrozenClock()
|
|
{
|
|
// Arrange & Act
|
|
var inputs = await _fixture.SnapshotInputsAsync();
|
|
|
|
// Assert
|
|
inputs.SnapshotTimestamp.Should().Be(_fixture.FrozenTimestamp, "Snapshot timestamp should match frozen clock");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region E2E-8200-014: Parallel Execution
|
|
|
|
[Fact(DisplayName = "10 concurrent pipeline runs produce identical results")]
|
|
public async Task FullPipeline_ParallelExecution_10Concurrent_AllIdentical()
|
|
{
|
|
// Arrange
|
|
var inputs = await _fixture.SnapshotInputsAsync();
|
|
const int concurrentRuns = 10;
|
|
|
|
// Act - Run pipeline 10 times in parallel
|
|
var tasks = Enumerable.Range(0, concurrentRuns)
|
|
.Select(_ => _fixture.RunFullPipelineAsync(inputs))
|
|
.ToList();
|
|
|
|
var results = await Task.WhenAll(tasks);
|
|
|
|
// Assert - All results must be identical
|
|
var comparison = ManifestComparer.CompareMultiple(results.ToList());
|
|
comparison.AllMatch.Should().BeTrue($"All {concurrentRuns} concurrent runs must produce identical results. {comparison.Summary}");
|
|
}
|
|
|
|
[Fact(DisplayName = "5 concurrent pipeline runs produce identical verdict IDs")]
|
|
public async Task FullPipeline_ParallelExecution_5Concurrent_IdenticalVerdictIds()
|
|
{
|
|
// Arrange
|
|
var inputs = await _fixture.SnapshotInputsAsync();
|
|
const int concurrentRuns = 5;
|
|
|
|
// Act - Run pipeline 5 times in parallel
|
|
var tasks = Enumerable.Range(0, concurrentRuns)
|
|
.Select(_ => _fixture.RunFullPipelineAsync(inputs))
|
|
.ToList();
|
|
|
|
var results = await Task.WhenAll(tasks);
|
|
|
|
// Assert - All verdict IDs must match
|
|
var firstVerdictId = results[0].VerdictId;
|
|
foreach (var result in results)
|
|
{
|
|
result.VerdictId.Should().Be(firstVerdictId, "All parallel runs must produce same verdict ID");
|
|
}
|
|
}
|
|
|
|
[Fact(DisplayName = "Parallel runs with VEX exceptions produce identical results")]
|
|
public async Task FullPipeline_ParallelWithVex_ProducesIdenticalResults()
|
|
{
|
|
// Arrange - Create inputs with VEX exceptions
|
|
var vexDocument = E2EReproducibilityTestFixture.CreateVexDocumentWithExceptions("CVE-2024-0001");
|
|
var inputs = await _fixture.SnapshotInputsAsync(vexDocumentPath: null);
|
|
var inputsWithVex = new InputSnapshot
|
|
{
|
|
Sbom = inputs.Sbom,
|
|
SbomHash = inputs.SbomHash,
|
|
AdvisoryFeed = inputs.AdvisoryFeed,
|
|
AdvisoryFeedHash = inputs.AdvisoryFeedHash,
|
|
PolicyPack = inputs.PolicyPack,
|
|
PolicyPackHash = inputs.PolicyPackHash,
|
|
VexDocument = vexDocument,
|
|
VexDocumentHash = E2EReproducibilityTestFixture.ComputeHash(vexDocument),
|
|
SnapshotTimestamp = inputs.SnapshotTimestamp
|
|
};
|
|
|
|
const int concurrentRuns = 5;
|
|
|
|
// Act - Run pipeline 5 times in parallel
|
|
var tasks = Enumerable.Range(0, concurrentRuns)
|
|
.Select(_ => _fixture.RunFullPipelineAsync(inputsWithVex))
|
|
.ToList();
|
|
|
|
var results = await Task.WhenAll(tasks);
|
|
|
|
// Assert - All results must be identical
|
|
var comparison = ManifestComparer.CompareMultiple(results.ToList());
|
|
comparison.AllMatch.Should().BeTrue("All parallel runs with VEX must produce identical results");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Edge Cases and Error Handling
|
|
|
|
[Fact(DisplayName = "Empty SBOM produces deterministic empty result")]
|
|
public async Task FullPipeline_EmptySbom_ProducesDeterministicResult()
|
|
{
|
|
// Arrange - Create empty SBOM
|
|
var emptySbom = System.Text.Encoding.UTF8.GetBytes(
|
|
System.Text.Json.JsonSerializer.Serialize(new { bomFormat = "CycloneDX", specVersion = "1.5", version = 1, components = Array.Empty<object>() }));
|
|
|
|
var inputs = new InputSnapshot
|
|
{
|
|
Sbom = emptySbom,
|
|
SbomHash = E2EReproducibilityTestFixture.ComputeHash(emptySbom),
|
|
AdvisoryFeed = E2EReproducibilityTestFixture.CreateMockAdvisoryFeed(),
|
|
AdvisoryFeedHash = E2EReproducibilityTestFixture.ComputeHash(E2EReproducibilityTestFixture.CreateMockAdvisoryFeed()),
|
|
PolicyPack = E2EReproducibilityTestFixture.CreateDefaultPolicyPack(),
|
|
PolicyPackHash = E2EReproducibilityTestFixture.ComputeHash(E2EReproducibilityTestFixture.CreateDefaultPolicyPack()),
|
|
VexDocument = null,
|
|
VexDocumentHash = null,
|
|
SnapshotTimestamp = _fixture.FrozenTimestamp
|
|
};
|
|
|
|
// Act - Run pipeline twice
|
|
var result1 = await _fixture.RunFullPipelineAsync(inputs);
|
|
var result2 = await _fixture.RunFullPipelineAsync(inputs);
|
|
|
|
// Assert - Results must be identical even with empty SBOM
|
|
result1.VerdictId.Should().Be(result2.VerdictId);
|
|
result1.BundleManifestHash.Should().Be(result2.BundleManifestHash);
|
|
}
|
|
|
|
[Fact(DisplayName = "VEX exceptions reduce blocking findings deterministically")]
|
|
public async Task FullPipeline_VexExceptions_ReduceBlockingFindingsDeterministically()
|
|
{
|
|
// Arrange - Run without VEX
|
|
var inputsWithoutVex = await _fixture.SnapshotInputsAsync();
|
|
var resultWithoutVex = await _fixture.RunFullPipelineAsync(inputsWithoutVex);
|
|
|
|
// Run with VEX exception for CVE-2024-0001 (CRITICAL)
|
|
var vexDocument = E2EReproducibilityTestFixture.CreateVexDocumentWithExceptions("CVE-2024-0001");
|
|
var inputsWithVex = new InputSnapshot
|
|
{
|
|
Sbom = inputsWithoutVex.Sbom,
|
|
SbomHash = inputsWithoutVex.SbomHash,
|
|
AdvisoryFeed = inputsWithoutVex.AdvisoryFeed,
|
|
AdvisoryFeedHash = inputsWithoutVex.AdvisoryFeedHash,
|
|
PolicyPack = inputsWithoutVex.PolicyPack,
|
|
PolicyPackHash = inputsWithoutVex.PolicyPackHash,
|
|
VexDocument = vexDocument,
|
|
VexDocumentHash = E2EReproducibilityTestFixture.ComputeHash(vexDocument),
|
|
SnapshotTimestamp = inputsWithoutVex.SnapshotTimestamp
|
|
};
|
|
|
|
var resultWithVex = await _fixture.RunFullPipelineAsync(inputsWithVex);
|
|
|
|
// Assert - VEX should change the verdict
|
|
resultWithVex.VerdictId.Should().NotBe(resultWithoutVex.VerdictId, "VEX exception should change verdict");
|
|
|
|
// But the result with VEX should be deterministic
|
|
var resultWithVex2 = await _fixture.RunFullPipelineAsync(inputsWithVex);
|
|
resultWithVex.VerdictId.Should().Be(resultWithVex2.VerdictId, "VEX result should be deterministic");
|
|
}
|
|
|
|
[Fact(DisplayName = "DSSE envelope hash is deterministic")]
|
|
public async Task DsseEnvelope_Hash_IsDeterministic()
|
|
{
|
|
// Arrange
|
|
var inputs = await _fixture.SnapshotInputsAsync();
|
|
|
|
// Act - Run pipeline 3 times
|
|
var result1 = await _fixture.RunFullPipelineAsync(inputs);
|
|
var result2 = await _fixture.RunFullPipelineAsync(inputs);
|
|
var result3 = await _fixture.RunFullPipelineAsync(inputs);
|
|
|
|
// Assert - All envelope hashes must match
|
|
result1.EnvelopeHash.Should().Be(result2.EnvelopeHash, "Envelope hash run 1 vs 2");
|
|
result2.EnvelopeHash.Should().Be(result3.EnvelopeHash, "Envelope hash run 2 vs 3");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Comparison Helper Tests
|
|
|
|
[Fact(DisplayName = "ManifestComparer generates readable diff report")]
|
|
public async Task ManifestComparer_GeneratesReadableDiffReport()
|
|
{
|
|
// Arrange
|
|
var inputs1 = await _fixture.SnapshotInputsAsync();
|
|
|
|
// Create different inputs
|
|
var differentSbom = System.Text.Encoding.UTF8.GetBytes(
|
|
System.Text.Json.JsonSerializer.Serialize(new
|
|
{
|
|
bomFormat = "CycloneDX",
|
|
specVersion = "1.5",
|
|
version = 1,
|
|
components = new[] { new { name = "different", version = "1.0.0", purl = "pkg:npm/different@1.0.0" } }
|
|
}));
|
|
|
|
var inputs2 = new InputSnapshot
|
|
{
|
|
Sbom = differentSbom,
|
|
SbomHash = E2EReproducibilityTestFixture.ComputeHash(differentSbom),
|
|
AdvisoryFeed = inputs1.AdvisoryFeed,
|
|
AdvisoryFeedHash = inputs1.AdvisoryFeedHash,
|
|
PolicyPack = inputs1.PolicyPack,
|
|
PolicyPackHash = inputs1.PolicyPackHash,
|
|
VexDocument = null,
|
|
VexDocumentHash = null,
|
|
SnapshotTimestamp = inputs1.SnapshotTimestamp
|
|
};
|
|
|
|
// Act
|
|
var result1 = await _fixture.RunFullPipelineAsync(inputs1);
|
|
var result2 = await _fixture.RunFullPipelineAsync(inputs2);
|
|
var comparison = ManifestComparer.Compare(result1, result2);
|
|
var report = ManifestComparer.GenerateDiffReport(comparison);
|
|
|
|
// Assert
|
|
comparison.IsMatch.Should().BeFalse();
|
|
report.Should().Contain("difference");
|
|
report.Should().Contain("VerdictId");
|
|
}
|
|
|
|
[Fact(DisplayName = "ManifestComparer multiple comparison returns correct summary")]
|
|
public async Task ManifestComparer_MultipleComparison_ReturnsCorrectSummary()
|
|
{
|
|
// Arrange
|
|
var inputs = await _fixture.SnapshotInputsAsync();
|
|
|
|
// Act - Run pipeline 3 times
|
|
var results = new List<PipelineResult>
|
|
{
|
|
await _fixture.RunFullPipelineAsync(inputs),
|
|
await _fixture.RunFullPipelineAsync(inputs),
|
|
await _fixture.RunFullPipelineAsync(inputs)
|
|
};
|
|
|
|
var comparison = ManifestComparer.CompareMultiple(results);
|
|
|
|
// Assert
|
|
comparison.AllMatch.Should().BeTrue();
|
|
comparison.Summary.Should().Contain("identical");
|
|
}
|
|
|
|
#endregion
|
|
}
|
|
|
|
/// <summary>
|
|
/// Collection definition for E2E reproducibility tests to share the fixture.
|
|
/// </summary>
|
|
[CollectionDefinition("E2EReproducibility")]
|
|
public sealed class E2EReproducibilityCollection : ICollectionFixture<E2EReproducibilityTestFixture>
|
|
{
|
|
}
|