// ----------------------------------------------------------------------------- // VerdictArtifactDeterminismTests.cs // Sprint: SPRINT_5100_0009_0001 - Scanner Module Test Implementation // Task: SCANNER-5100-010 - Expand determinism tests: verdict artifact payload hash stable // Description: Tests to validate verdict artifact generation determinism // ----------------------------------------------------------------------------- using System.Text; using FluentAssertions; using StellaOps.Canonical.Json; using StellaOps.Testing.Determinism; using Xunit; namespace StellaOps.Integration.Determinism; /// /// Determinism validation tests for verdict artifact generation. /// Ensures identical inputs produce identical verdict artifacts across: /// - Multiple runs with frozen time /// - Parallel execution /// - Change ordering /// - Proof spine integration /// public class VerdictArtifactDeterminismTests { #region Basic Determinism Tests [Fact] public void VerdictArtifact_WithIdenticalInput_ProducesDeterministicOutput() { // Arrange var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z"); var input = CreateSampleVerdictInput(); // Act - Generate verdict artifact multiple times var verdict1 = GenerateVerdictArtifact(input, frozenTime); var verdict2 = GenerateVerdictArtifact(input, frozenTime); var verdict3 = GenerateVerdictArtifact(input, frozenTime); // Serialize to canonical JSON var json1 = CanonJson.Serialize(verdict1); var json2 = CanonJson.Serialize(verdict2); var json3 = CanonJson.Serialize(verdict3); // Assert - All outputs should be identical json1.Should().Be(json2); json2.Should().Be(json3); } [Fact] public void VerdictArtifact_CanonicalHash_IsStable() { // Arrange var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z"); var input = CreateSampleVerdictInput(); // Act - Generate verdict and compute canonical hash twice var verdict1 = GenerateVerdictArtifact(input, frozenTime); var hash1 = ComputeCanonicalHash(verdict1); var verdict2 = GenerateVerdictArtifact(input, frozenTime); var hash2 = ComputeCanonicalHash(verdict2); // Assert hash1.Should().Be(hash2, "Same input should produce same canonical hash"); hash1.Should().MatchRegex("^[0-9a-f]{64}$"); } [Fact] public void VerdictArtifact_DeterminismManifest_CanBeCreated() { // Arrange var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z"); var input = CreateSampleVerdictInput(); var verdict = GenerateVerdictArtifact(input, frozenTime); var verdictBytes = Encoding.UTF8.GetBytes(CanonJson.Serialize(verdict)); var artifactInfo = new ArtifactInfo { Type = "verdict-artifact", Name = "test-delta-verdict", Version = "1.0.0", Format = "delta-verdict@1.0" }; var toolchain = new ToolchainInfo { Platform = ".NET 10.0", Components = new[] { new ComponentInfo { Name = "StellaOps.Scanner.SmartDiff", Version = "1.0.0" } } }; // Act - Create determinism manifest var manifest = DeterminismManifestWriter.CreateManifest( verdictBytes, artifactInfo, toolchain); // Assert manifest.SchemaVersion.Should().Be("1.0"); manifest.Artifact.Format.Should().Be("delta-verdict@1.0"); manifest.CanonicalHash.Algorithm.Should().Be("SHA-256"); manifest.CanonicalHash.Value.Should().MatchRegex("^[0-9a-f]{64}$"); } [Fact] public async Task VerdictArtifact_ParallelGeneration_ProducesDeterministicOutput() { // Arrange var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z"); var input = CreateSampleVerdictInput(); // Act - Generate in parallel 20 times var tasks = Enumerable.Range(0, 20) .Select(_ => Task.Run(() => CanonJson.Serialize(GenerateVerdictArtifact(input, frozenTime)))) .ToArray(); var verdicts = await Task.WhenAll(tasks); // Assert - All outputs should be identical verdicts.Should().AllBe(verdicts[0]); } #endregion #region Change Ordering Tests [Fact] public void VerdictArtifact_ChangesAreDeterministicallyOrdered() { // Arrange - Create input with changes in random order var changes = new[] { CreateChange("CVE-2024-0003", "pkg:npm/c@1.0.0", "new"), CreateChange("CVE-2024-0001", "pkg:npm/a@1.0.0", "resolved"), CreateChange("CVE-2024-0002", "pkg:npm/b@1.0.0", "severity_changed") }; var input = new VerdictInput { VerdictId = Guid.Parse("11111111-1111-1111-1111-111111111111"), BaselineScanId = Guid.Parse("00000000-0000-0000-0000-000000000001"), CurrentScanId = Guid.Parse("00000000-0000-0000-0000-000000000002"), Changes = changes }; var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z"); // Act var verdict1 = GenerateVerdictArtifact(input, frozenTime); var verdict2 = GenerateVerdictArtifact(input, frozenTime); // Assert - Outputs should be identical var json1 = CanonJson.Serialize(verdict1); var json2 = CanonJson.Serialize(verdict2); json1.Should().Be(json2); // Verify changes are sorted by CVE ID, then by package URL for (int i = 1; i < verdict1.Changes.Count; i++) { var cmp = string.CompareOrdinal(verdict1.Changes[i - 1].CveId, verdict1.Changes[i].CveId); if (cmp == 0) { cmp = string.CompareOrdinal(verdict1.Changes[i - 1].PackageUrl, verdict1.Changes[i].PackageUrl); } cmp.Should().BeLessOrEqualTo(0, "Changes should be sorted by CVE ID, then package URL"); } } [Fact] public void VerdictArtifact_ChangesWithSameCveAndPackage_SortedByChangeType() { // Arrange - Multiple changes for same CVE/package var changes = new[] { CreateChange("CVE-2024-0001", "pkg:npm/test@1.0.0", "severity_changed"), CreateChange("CVE-2024-0001", "pkg:npm/test@1.0.0", "status_changed"), CreateChange("CVE-2024-0001", "pkg:npm/test@1.0.0", "epss_changed") }; var input = new VerdictInput { VerdictId = Guid.Parse("22222222-2222-2222-2222-222222222222"), BaselineScanId = Guid.Parse("00000000-0000-0000-0000-000000000001"), CurrentScanId = Guid.Parse("00000000-0000-0000-0000-000000000002"), Changes = changes }; var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z"); // Act var verdict1 = GenerateVerdictArtifact(input, frozenTime); var verdict2 = GenerateVerdictArtifact(input, frozenTime); // Assert var json1 = CanonJson.Serialize(verdict1); var json2 = CanonJson.Serialize(verdict2); json1.Should().Be(json2); } #endregion #region Change Type Tests [Theory] [InlineData("new")] [InlineData("resolved")] [InlineData("severity_changed")] [InlineData("status_changed")] [InlineData("epss_changed")] [InlineData("reachability_changed")] [InlineData("vex_status_changed")] public void VerdictArtifact_ChangeTypeIsPreserved(string changeType) { // Arrange var change = CreateChange("CVE-2024-0001", "pkg:npm/test@1.0.0", changeType); var input = new VerdictInput { VerdictId = Guid.Parse("33333333-3333-3333-3333-333333333333"), BaselineScanId = Guid.Parse("00000000-0000-0000-0000-000000000001"), CurrentScanId = Guid.Parse("00000000-0000-0000-0000-000000000002"), Changes = new[] { change } }; var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z"); // Act var verdict = GenerateVerdictArtifact(input, frozenTime); // Assert verdict.Changes[0].ChangeType.Should().Be(changeType); } #endregion #region Proof Spine Tests [Fact] public void VerdictArtifact_ProofSpinesAreDeterministicallyOrdered() { // Arrange var changes = new[] { CreateChange("CVE-2024-0001", "pkg:npm/a@1.0.0", "new") with { ProofSpine = new ProofSpine { SpineId = "spine-a", Evidences = new[] { CreateProofEvidence("epss", 0.8), CreateProofEvidence("reachability", 0.9), CreateProofEvidence("vex", 1.0) } } } }; var input = new VerdictInput { VerdictId = Guid.Parse("44444444-4444-4444-4444-444444444444"), BaselineScanId = Guid.Parse("00000000-0000-0000-0000-000000000001"), CurrentScanId = Guid.Parse("00000000-0000-0000-0000-000000000002"), Changes = changes }; var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z"); // Act var verdict1 = GenerateVerdictArtifact(input, frozenTime); var verdict2 = GenerateVerdictArtifact(input, frozenTime); // Assert var json1 = CanonJson.Serialize(verdict1); var json2 = CanonJson.Serialize(verdict2); json1.Should().Be(json2); // Verify evidences in proof spine are sorted var evidences = verdict1.Changes[0].ProofSpine!.Evidences; for (int i = 1; i < evidences.Count; i++) { string.CompareOrdinal(evidences[i - 1].EvidenceType, evidences[i].EvidenceType) .Should().BeLessOrEqualTo(0, "Proof spine evidences should be sorted by type"); } } [Fact] public void VerdictArtifact_ProofSpineHashIsStable() { // Arrange var change = CreateChange("CVE-2024-0001", "pkg:npm/test@1.0.0", "new") with { ProofSpine = new ProofSpine { SpineId = "spine-test", Evidences = new[] { CreateProofEvidence("epss", 0.5), CreateProofEvidence("reachability", 0.75) } } }; var input = new VerdictInput { VerdictId = Guid.Parse("55555555-5555-5555-5555-555555555555"), BaselineScanId = Guid.Parse("00000000-0000-0000-0000-000000000001"), CurrentScanId = Guid.Parse("00000000-0000-0000-0000-000000000002"), Changes = new[] { change } }; var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z"); // Act var verdict1 = GenerateVerdictArtifact(input, frozenTime); var verdict2 = GenerateVerdictArtifact(input, frozenTime); // Assert verdict1.Changes[0].ProofSpine!.SpineHash.Should().Be(verdict2.Changes[0].ProofSpine!.SpineHash); verdict1.Changes[0].ProofSpine!.SpineHash.Should().MatchRegex("^[0-9a-f]{64}$"); } #endregion #region Summary Statistics Tests [Fact] public void VerdictArtifact_SummaryStatisticsAreDeterministic() { // Arrange var changes = new[] { CreateChange("CVE-2024-0001", "pkg:npm/a@1.0.0", "new"), CreateChange("CVE-2024-0002", "pkg:npm/b@1.0.0", "new"), CreateChange("CVE-2024-0003", "pkg:npm/c@1.0.0", "resolved"), CreateChange("CVE-2024-0004", "pkg:npm/d@1.0.0", "severity_changed") }; var input = new VerdictInput { VerdictId = Guid.Parse("66666666-6666-6666-6666-666666666666"), BaselineScanId = Guid.Parse("00000000-0000-0000-0000-000000000001"), CurrentScanId = Guid.Parse("00000000-0000-0000-0000-000000000002"), Changes = changes }; var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z"); // Act var verdict1 = GenerateVerdictArtifact(input, frozenTime); var verdict2 = GenerateVerdictArtifact(input, frozenTime); // Assert verdict1.Summary.Should().NotBeNull(); verdict1.Summary.TotalChanges.Should().Be(verdict2.Summary.TotalChanges); verdict1.Summary.NewFindings.Should().Be(verdict2.Summary.NewFindings); verdict1.Summary.ResolvedFindings.Should().Be(verdict2.Summary.ResolvedFindings); } #endregion #region Empty/Edge Case Tests [Fact] public void VerdictArtifact_NoChanges_ProducesDeterministicOutput() { // Arrange var input = new VerdictInput { VerdictId = Guid.Parse("77777777-7777-7777-7777-777777777777"), BaselineScanId = Guid.Parse("00000000-0000-0000-0000-000000000001"), CurrentScanId = Guid.Parse("00000000-0000-0000-0000-000000000002"), Changes = Array.Empty() }; var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z"); // Act var hash1 = ComputeCanonicalHash(GenerateVerdictArtifact(input, frozenTime)); var hash2 = ComputeCanonicalHash(GenerateVerdictArtifact(input, frozenTime)); // Assert hash1.Should().Be(hash2); } [Fact] public void VerdictArtifact_ManyChanges_ProducesDeterministicOutput() { // Arrange - Create 500 changes var changes = Enumerable.Range(0, 500) .Select(i => CreateChange( $"CVE-2024-{i:D4}", $"pkg:npm/package-{i}@1.0.0", i % 3 == 0 ? "new" : i % 2 == 0 ? "resolved" : "severity_changed")) .ToArray(); var input = new VerdictInput { VerdictId = Guid.Parse("88888888-8888-8888-8888-888888888888"), BaselineScanId = Guid.Parse("00000000-0000-0000-0000-000000000001"), CurrentScanId = Guid.Parse("00000000-0000-0000-0000-000000000002"), Changes = changes }; var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z"); // Act var hash1 = ComputeCanonicalHash(GenerateVerdictArtifact(input, frozenTime)); var hash2 = ComputeCanonicalHash(GenerateVerdictArtifact(input, frozenTime)); // Assert hash1.Should().Be(hash2); } #endregion #region Helper Methods private static VerdictInput CreateSampleVerdictInput() { return new VerdictInput { VerdictId = Guid.Parse("99999999-9999-9999-9999-999999999999"), BaselineScanId = Guid.Parse("00000000-0000-0000-0000-000000000001"), CurrentScanId = Guid.Parse("00000000-0000-0000-0000-000000000002"), Changes = new[] { CreateChange("CVE-2024-1234", "pkg:npm/lodash@4.17.20", "new"), CreateChange("CVE-2024-5678", "pkg:npm/axios@0.21.0", "resolved"), CreateChange("CVE-2024-9012", "pkg:npm/express@4.17.1", "severity_changed") } }; } private static VerdictChange CreateChange(string cveId, string packageUrl, string changeType) { return new VerdictChange { CveId = cveId, PackageUrl = packageUrl, ChangeType = changeType, ProofSpine = null }; } private static ProofEvidence CreateProofEvidence(string evidenceType, double confidence) { return new ProofEvidence { EvidenceType = evidenceType, Confidence = confidence, Summary = $"{evidenceType} evidence" }; } private static VerdictArtifact GenerateVerdictArtifact(VerdictInput input, DateTimeOffset timestamp) { // Sort changes deterministically var sortedChanges = input.Changes .OrderBy(c => c.CveId, StringComparer.Ordinal) .ThenBy(c => c.PackageUrl, StringComparer.Ordinal) .ThenBy(c => c.ChangeType, StringComparer.Ordinal) .Select(c => new VerdictChangeOutput { CveId = c.CveId, PackageUrl = c.PackageUrl, ChangeType = c.ChangeType, ProofSpine = c.ProofSpine != null ? ProcessProofSpine(c.ProofSpine) : null }) .ToList(); // Compute summary statistics var summary = new VerdictSummary { TotalChanges = sortedChanges.Count, NewFindings = sortedChanges.Count(c => c.ChangeType == "new"), ResolvedFindings = sortedChanges.Count(c => c.ChangeType == "resolved"), OtherChanges = sortedChanges.Count(c => c.ChangeType != "new" && c.ChangeType != "resolved") }; return new VerdictArtifact { VerdictId = input.VerdictId, BaselineScanId = input.BaselineScanId, CurrentScanId = input.CurrentScanId, Timestamp = timestamp, Changes = sortedChanges, Summary = summary }; } private static ProofSpineOutput ProcessProofSpine(ProofSpine spine) { var sortedEvidences = spine.Evidences .OrderBy(e => e.EvidenceType, StringComparer.Ordinal) .ToList(); // Compute spine hash from sorted evidences var evidenceJson = CanonJson.Serialize(sortedEvidences); var spineHash = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(evidenceJson)); return new ProofSpineOutput { SpineId = spine.SpineId, Evidences = sortedEvidences, SpineHash = spineHash }; } private static string ComputeCanonicalHash(VerdictArtifact artifact) { var json = CanonJson.Serialize(artifact); return CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(json)); } #endregion #region DTOs private sealed record VerdictInput { public required Guid VerdictId { get; init; } public required Guid BaselineScanId { get; init; } public required Guid CurrentScanId { get; init; } public required VerdictChange[] Changes { get; init; } } private sealed record VerdictChange { public required string CveId { get; init; } public required string PackageUrl { get; init; } public required string ChangeType { get; init; } public ProofSpine? ProofSpine { get; init; } } private sealed record ProofSpine { public required string SpineId { get; init; } public required ProofEvidence[] Evidences { get; init; } } private sealed record ProofEvidence { public required string EvidenceType { get; init; } public required double Confidence { get; init; } public required string Summary { get; init; } } private sealed record VerdictArtifact { public required Guid VerdictId { get; init; } public required Guid BaselineScanId { get; init; } public required Guid CurrentScanId { get; init; } public required DateTimeOffset Timestamp { get; init; } public required IReadOnlyList Changes { get; init; } public required VerdictSummary Summary { get; init; } } private sealed record VerdictChangeOutput { public required string CveId { get; init; } public required string PackageUrl { get; init; } public required string ChangeType { get; init; } public ProofSpineOutput? ProofSpine { get; init; } } private sealed record ProofSpineOutput { public required string SpineId { get; init; } public required IReadOnlyList Evidences { get; init; } public required string SpineHash { get; init; } } private sealed record VerdictSummary { public required int TotalChanges { get; init; } public required int NewFindings { get; init; } public required int ResolvedFindings { get; init; } public required int OtherChanges { get; init; } } #endregion }