// ----------------------------------------------------------------------------- // E2EReproducibilityTests.cs // Sprint: SPRINT_8200_0001_0004_e2e_reproducibility_test // Tasks: E2E-8200-011 to E2E-8200-014 - Reproducibility Tests // Description: End-to-end tests verifying full pipeline reproducibility. // Validates: identical verdict hash, identical manifest, frozen timestamps, // parallel execution produces identical results. // ----------------------------------------------------------------------------- using FluentAssertions; using Xunit; namespace StellaOps.Integration.E2E; /// /// End-to-end reproducibility tests for the full security scanning pipeline. /// Verifies that identical inputs always produce identical outputs across: /// - Sequential runs /// - Parallel runs /// - With frozen timestamps /// [Collection("E2EReproducibility")] [Trait("Category", "Integration")] [Trait("Sprint", "8200.0001.0004")] [Trait("Feature", "E2E-Reproducibility")] public sealed class E2EReproducibilityTests : IClassFixture, IAsyncLifetime { private readonly E2EReproducibilityTestFixture _fixture; public E2EReproducibilityTests(E2EReproducibilityTestFixture fixture) { _fixture = fixture; } public async Task InitializeAsync() { await _fixture.InitializeAsync(); } public Task DisposeAsync() => Task.CompletedTask; #region E2E-8200-011: Identical Verdict Hash [Fact(DisplayName = "Pipeline produces identical verdict hash across runs")] public async Task FullPipeline_ProducesIdenticalVerdictHash_AcrossRuns() { // Arrange - Create input snapshot var inputs = await _fixture.SnapshotInputsAsync(); // Act - Run pipeline twice with identical inputs var result1 = await _fixture.RunFullPipelineAsync(inputs); var result2 = await _fixture.RunFullPipelineAsync(inputs); // Assert - Verdict IDs must match result1.VerdictId.Should().NotBeNullOrEmpty("Verdict ID should be computed"); result2.VerdictId.Should().NotBeNullOrEmpty("Verdict ID should be computed"); result1.VerdictId.Should().Be(result2.VerdictId, "Verdict ID must be identical across runs"); // Verdict hash must match result1.VerdictHash.Should().Be(result2.VerdictHash, "Verdict hash must be identical"); } [Fact(DisplayName = "Pipeline produces identical verdict hash with 5 sequential runs")] public async Task FullPipeline_ProducesIdenticalVerdictHash_With5SequentialRuns() { // Arrange var inputs = await _fixture.SnapshotInputsAsync(); var results = new List(); // Act - Run pipeline 5 times sequentially for (int i = 0; i < 5; i++) { results.Add(await _fixture.RunFullPipelineAsync(inputs)); } // Assert - All verdict IDs must match var firstVerdictId = results[0].VerdictId; foreach (var result in results) { result.VerdictId.Should().Be(firstVerdictId, $"Run {results.IndexOf(result) + 1} verdict ID must match first run"); } } [Fact(DisplayName = "Verdict ID format is content-addressed")] public async Task VerdictId_Format_IsContentAddressed() { // Arrange var inputs = await _fixture.SnapshotInputsAsync(); // Act var result = await _fixture.RunFullPipelineAsync(inputs); // Assert - Verdict ID should be in content-addressed format result.VerdictId.Should().StartWith("verdict:sha256:", "Verdict ID must use sha256 content-addressing"); result.VerdictId.Should().MatchRegex(@"^verdict:sha256:[0-9a-f]{64}$", "Verdict ID must be valid sha256 hex"); } #endregion #region E2E-8200-012: Identical Bundle Manifest [Fact(DisplayName = "Pipeline produces identical bundle manifest across runs")] public async Task FullPipeline_ProducesIdenticalBundleManifest_AcrossRuns() { // Arrange var inputs = await _fixture.SnapshotInputsAsync(); // Act - Run pipeline twice var result1 = await _fixture.RunFullPipelineAsync(inputs); var result2 = await _fixture.RunFullPipelineAsync(inputs); // Assert - Bundle manifests must be byte-for-byte identical result1.BundleManifest.Should().BeEquivalentTo(result2.BundleManifest, "Bundle manifest bytes must match"); result1.BundleManifestHash.Should().Be(result2.BundleManifestHash, "Bundle manifest hash must match"); } [Fact(DisplayName = "Bundle manifest contains all artifact hashes")] public async Task BundleManifest_ContainsAllArtifactHashes() { // Arrange var inputs = await _fixture.SnapshotInputsAsync(); // Act var result = await _fixture.RunFullPipelineAsync(inputs); // Assert - Parse manifest and verify structure var manifestJson = System.Text.Encoding.UTF8.GetString(result.BundleManifest); using var doc = System.Text.Json.JsonDocument.Parse(manifestJson); var root = doc.RootElement; root.TryGetProperty("version", out _).Should().BeTrue("Manifest should have version"); root.TryGetProperty("createdAt", out _).Should().BeTrue("Manifest should have createdAt"); root.TryGetProperty("artifacts", out var artifacts).Should().BeTrue("Manifest should have artifacts"); artifacts.TryGetProperty("sbom", out _).Should().BeTrue("Artifacts should include SBOM hash"); artifacts.TryGetProperty("advisory-feed", out _).Should().BeTrue("Artifacts should include advisory feed hash"); artifacts.TryGetProperty("policy-pack", out _).Should().BeTrue("Artifacts should include policy pack hash"); artifacts.TryGetProperty("envelope", out _).Should().BeTrue("Artifacts should include envelope hash"); } [Fact(DisplayName = "Manifest comparison detects differences")] public async Task ManifestComparer_DetectsDifferences_WhenInputsChange() { // Arrange - Create two different input snapshots var inputs1 = await _fixture.SnapshotInputsAsync(); // Modify SBOM to create different input var modifiedSbom = E2EReproducibilityTestFixture.CreateMinimalSbom(); var sbomJson = System.Text.Encoding.UTF8.GetString(modifiedSbom); var modifiedSbomJson = sbomJson.Replace("4.17.20", "4.17.21"); // Change version var modifiedSbomBytes = System.Text.Encoding.UTF8.GetBytes(modifiedSbomJson); var inputs2 = new InputSnapshot { Sbom = modifiedSbomBytes, SbomHash = E2EReproducibilityTestFixture.ComputeHash(modifiedSbomBytes), AdvisoryFeed = inputs1.AdvisoryFeed, AdvisoryFeedHash = inputs1.AdvisoryFeedHash, PolicyPack = inputs1.PolicyPack, PolicyPackHash = inputs1.PolicyPackHash, VexDocument = inputs1.VexDocument, VexDocumentHash = inputs1.VexDocumentHash, SnapshotTimestamp = inputs1.SnapshotTimestamp }; // Act var result1 = await _fixture.RunFullPipelineAsync(inputs1); var result2 = await _fixture.RunFullPipelineAsync(inputs2); // Assert - Results should differ var comparison = ManifestComparer.Compare(result1, result2); comparison.IsMatch.Should().BeFalse("Different inputs should produce different outputs"); comparison.Differences.Should().NotBeEmpty("Should detect at least one difference"); } #endregion #region E2E-8200-013: Frozen Clock Timestamps [Fact(DisplayName = "Pipeline produces identical timestamps with frozen clock")] public async Task FullPipeline_ProducesIdenticalTimestamps_WithFrozenClock() { // Arrange var inputs = await _fixture.SnapshotInputsAsync(); // Act - Run pipeline twice var result1 = await _fixture.RunFullPipelineAsync(inputs); var result2 = await _fixture.RunFullPipelineAsync(inputs); // Assert - Execution timestamps must match (frozen clock) result1.ExecutionTimestamp.Should().Be(_fixture.FrozenTimestamp, "Timestamp should match frozen clock"); result2.ExecutionTimestamp.Should().Be(_fixture.FrozenTimestamp, "Timestamp should match frozen clock"); result1.ExecutionTimestamp.Should().Be(result2.ExecutionTimestamp, "Timestamps must be identical"); } [Fact(DisplayName = "Manifest createdAt matches frozen timestamp")] public async Task BundleManifest_CreatedAt_MatchesFrozenTimestamp() { // Arrange var inputs = await _fixture.SnapshotInputsAsync(); // Act var result = await _fixture.RunFullPipelineAsync(inputs); // Assert - Parse manifest and verify timestamp var manifestJson = System.Text.Encoding.UTF8.GetString(result.BundleManifest); using var doc = System.Text.Json.JsonDocument.Parse(manifestJson); var createdAt = doc.RootElement.GetProperty("createdAt").GetDateTimeOffset(); createdAt.Should().Be(_fixture.FrozenTimestamp, "Manifest createdAt should match frozen clock"); } [Fact(DisplayName = "Input snapshot timestamp matches frozen clock")] public async Task InputSnapshot_Timestamp_MatchesFrozenClock() { // Arrange & Act var inputs = await _fixture.SnapshotInputsAsync(); // Assert inputs.SnapshotTimestamp.Should().Be(_fixture.FrozenTimestamp, "Snapshot timestamp should match frozen clock"); } #endregion #region E2E-8200-014: Parallel Execution [Fact(DisplayName = "10 concurrent pipeline runs produce identical results")] public async Task FullPipeline_ParallelExecution_10Concurrent_AllIdentical() { // Arrange var inputs = await _fixture.SnapshotInputsAsync(); const int concurrentRuns = 10; // Act - Run pipeline 10 times in parallel var tasks = Enumerable.Range(0, concurrentRuns) .Select(_ => _fixture.RunFullPipelineAsync(inputs)) .ToList(); var results = await Task.WhenAll(tasks); // Assert - All results must be identical var comparison = ManifestComparer.CompareMultiple(results.ToList()); comparison.AllMatch.Should().BeTrue($"All {concurrentRuns} concurrent runs must produce identical results. {comparison.Summary}"); } [Fact(DisplayName = "5 concurrent pipeline runs produce identical verdict IDs")] public async Task FullPipeline_ParallelExecution_5Concurrent_IdenticalVerdictIds() { // Arrange var inputs = await _fixture.SnapshotInputsAsync(); const int concurrentRuns = 5; // Act - Run pipeline 5 times in parallel var tasks = Enumerable.Range(0, concurrentRuns) .Select(_ => _fixture.RunFullPipelineAsync(inputs)) .ToList(); var results = await Task.WhenAll(tasks); // Assert - All verdict IDs must match var firstVerdictId = results[0].VerdictId; foreach (var result in results) { result.VerdictId.Should().Be(firstVerdictId, "All parallel runs must produce same verdict ID"); } } [Fact(DisplayName = "Parallel runs with VEX exceptions produce identical results")] public async Task FullPipeline_ParallelWithVex_ProducesIdenticalResults() { // Arrange - Create inputs with VEX exceptions var vexDocument = E2EReproducibilityTestFixture.CreateVexDocumentWithExceptions("CVE-2024-0001"); var inputs = await _fixture.SnapshotInputsAsync(vexDocumentPath: null); var inputsWithVex = new InputSnapshot { Sbom = inputs.Sbom, SbomHash = inputs.SbomHash, AdvisoryFeed = inputs.AdvisoryFeed, AdvisoryFeedHash = inputs.AdvisoryFeedHash, PolicyPack = inputs.PolicyPack, PolicyPackHash = inputs.PolicyPackHash, VexDocument = vexDocument, VexDocumentHash = E2EReproducibilityTestFixture.ComputeHash(vexDocument), SnapshotTimestamp = inputs.SnapshotTimestamp }; const int concurrentRuns = 5; // Act - Run pipeline 5 times in parallel var tasks = Enumerable.Range(0, concurrentRuns) .Select(_ => _fixture.RunFullPipelineAsync(inputsWithVex)) .ToList(); var results = await Task.WhenAll(tasks); // Assert - All results must be identical var comparison = ManifestComparer.CompareMultiple(results.ToList()); comparison.AllMatch.Should().BeTrue("All parallel runs with VEX must produce identical results"); } #endregion #region Edge Cases and Error Handling [Fact(DisplayName = "Empty SBOM produces deterministic empty result")] public async Task FullPipeline_EmptySbom_ProducesDeterministicResult() { // Arrange - Create empty SBOM var emptySbom = System.Text.Encoding.UTF8.GetBytes( System.Text.Json.JsonSerializer.Serialize(new { bomFormat = "CycloneDX", specVersion = "1.5", version = 1, components = Array.Empty() })); var inputs = new InputSnapshot { Sbom = emptySbom, SbomHash = E2EReproducibilityTestFixture.ComputeHash(emptySbom), AdvisoryFeed = E2EReproducibilityTestFixture.CreateMockAdvisoryFeed(), AdvisoryFeedHash = E2EReproducibilityTestFixture.ComputeHash(E2EReproducibilityTestFixture.CreateMockAdvisoryFeed()), PolicyPack = E2EReproducibilityTestFixture.CreateDefaultPolicyPack(), PolicyPackHash = E2EReproducibilityTestFixture.ComputeHash(E2EReproducibilityTestFixture.CreateDefaultPolicyPack()), VexDocument = null, VexDocumentHash = null, SnapshotTimestamp = _fixture.FrozenTimestamp }; // Act - Run pipeline twice var result1 = await _fixture.RunFullPipelineAsync(inputs); var result2 = await _fixture.RunFullPipelineAsync(inputs); // Assert - Results must be identical even with empty SBOM result1.VerdictId.Should().Be(result2.VerdictId); result1.BundleManifestHash.Should().Be(result2.BundleManifestHash); } [Fact(DisplayName = "VEX exceptions reduce blocking findings deterministically")] public async Task FullPipeline_VexExceptions_ReduceBlockingFindingsDeterministically() { // Arrange - Run without VEX var inputsWithoutVex = await _fixture.SnapshotInputsAsync(); var resultWithoutVex = await _fixture.RunFullPipelineAsync(inputsWithoutVex); // Run with VEX exception for CVE-2024-0001 (CRITICAL) var vexDocument = E2EReproducibilityTestFixture.CreateVexDocumentWithExceptions("CVE-2024-0001"); var inputsWithVex = new InputSnapshot { Sbom = inputsWithoutVex.Sbom, SbomHash = inputsWithoutVex.SbomHash, AdvisoryFeed = inputsWithoutVex.AdvisoryFeed, AdvisoryFeedHash = inputsWithoutVex.AdvisoryFeedHash, PolicyPack = inputsWithoutVex.PolicyPack, PolicyPackHash = inputsWithoutVex.PolicyPackHash, VexDocument = vexDocument, VexDocumentHash = E2EReproducibilityTestFixture.ComputeHash(vexDocument), SnapshotTimestamp = inputsWithoutVex.SnapshotTimestamp }; var resultWithVex = await _fixture.RunFullPipelineAsync(inputsWithVex); // Assert - VEX should change the verdict resultWithVex.VerdictId.Should().NotBe(resultWithoutVex.VerdictId, "VEX exception should change verdict"); // But the result with VEX should be deterministic var resultWithVex2 = await _fixture.RunFullPipelineAsync(inputsWithVex); resultWithVex.VerdictId.Should().Be(resultWithVex2.VerdictId, "VEX result should be deterministic"); } [Fact(DisplayName = "DSSE envelope hash is deterministic")] public async Task DsseEnvelope_Hash_IsDeterministic() { // Arrange var inputs = await _fixture.SnapshotInputsAsync(); // Act - Run pipeline 3 times var result1 = await _fixture.RunFullPipelineAsync(inputs); var result2 = await _fixture.RunFullPipelineAsync(inputs); var result3 = await _fixture.RunFullPipelineAsync(inputs); // Assert - All envelope hashes must match result1.EnvelopeHash.Should().Be(result2.EnvelopeHash, "Envelope hash run 1 vs 2"); result2.EnvelopeHash.Should().Be(result3.EnvelopeHash, "Envelope hash run 2 vs 3"); } #endregion #region Comparison Helper Tests [Fact(DisplayName = "ManifestComparer generates readable diff report")] public async Task ManifestComparer_GeneratesReadableDiffReport() { // Arrange var inputs1 = await _fixture.SnapshotInputsAsync(); // Create different inputs var differentSbom = System.Text.Encoding.UTF8.GetBytes( System.Text.Json.JsonSerializer.Serialize(new { bomFormat = "CycloneDX", specVersion = "1.5", version = 1, components = new[] { new { name = "different", version = "1.0.0", purl = "pkg:npm/different@1.0.0" } } })); var inputs2 = new InputSnapshot { Sbom = differentSbom, SbomHash = E2EReproducibilityTestFixture.ComputeHash(differentSbom), AdvisoryFeed = inputs1.AdvisoryFeed, AdvisoryFeedHash = inputs1.AdvisoryFeedHash, PolicyPack = inputs1.PolicyPack, PolicyPackHash = inputs1.PolicyPackHash, VexDocument = null, VexDocumentHash = null, SnapshotTimestamp = inputs1.SnapshotTimestamp }; // Act var result1 = await _fixture.RunFullPipelineAsync(inputs1); var result2 = await _fixture.RunFullPipelineAsync(inputs2); var comparison = ManifestComparer.Compare(result1, result2); var report = ManifestComparer.GenerateDiffReport(comparison); // Assert comparison.IsMatch.Should().BeFalse(); report.Should().Contain("difference"); report.Should().Contain("VerdictId"); } [Fact(DisplayName = "ManifestComparer multiple comparison returns correct summary")] public async Task ManifestComparer_MultipleComparison_ReturnsCorrectSummary() { // Arrange var inputs = await _fixture.SnapshotInputsAsync(); // Act - Run pipeline 3 times var results = new List { await _fixture.RunFullPipelineAsync(inputs), await _fixture.RunFullPipelineAsync(inputs), await _fixture.RunFullPipelineAsync(inputs) }; var comparison = ManifestComparer.CompareMultiple(results); // Assert comparison.AllMatch.Should().BeTrue(); comparison.Summary.Should().Contain("identical"); } #endregion } /// /// Collection definition for E2E reproducibility tests to share the fixture. /// [CollectionDefinition("E2EReproducibility")] public sealed class E2EReproducibilityCollection : ICollectionFixture { }