// -----------------------------------------------------------------------------
// VerdictArtifactDeterminismTests.cs
// Sprint: SPRINT_5100_0009_0001 - Scanner Module Test Implementation
// Task: SCANNER-5100-010 - Expand determinism tests: verdict artifact payload hash stable
// Description: Tests to validate verdict artifact generation determinism
// -----------------------------------------------------------------------------
using System.Text;
using FluentAssertions;
using StellaOps.Canonical.Json;
using StellaOps.Testing.Determinism;
using Xunit;
namespace StellaOps.Integration.Determinism;
///
/// Determinism validation tests for verdict artifact generation.
/// Ensures identical inputs produce identical verdict artifacts across:
/// - Multiple runs with frozen time
/// - Parallel execution
/// - Change ordering
/// - Proof spine integration
///
public class VerdictArtifactDeterminismTests
{
#region Basic Determinism Tests
[Fact]
public void VerdictArtifact_WithIdenticalInput_ProducesDeterministicOutput()
{
// Arrange
var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z");
var input = CreateSampleVerdictInput();
// Act - Generate verdict artifact multiple times
var verdict1 = GenerateVerdictArtifact(input, frozenTime);
var verdict2 = GenerateVerdictArtifact(input, frozenTime);
var verdict3 = GenerateVerdictArtifact(input, frozenTime);
// Serialize to canonical JSON
var json1 = CanonJson.Serialize(verdict1);
var json2 = CanonJson.Serialize(verdict2);
var json3 = CanonJson.Serialize(verdict3);
// Assert - All outputs should be identical
json1.Should().Be(json2);
json2.Should().Be(json3);
}
[Fact]
public void VerdictArtifact_CanonicalHash_IsStable()
{
// Arrange
var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z");
var input = CreateSampleVerdictInput();
// Act - Generate verdict and compute canonical hash twice
var verdict1 = GenerateVerdictArtifact(input, frozenTime);
var hash1 = ComputeCanonicalHash(verdict1);
var verdict2 = GenerateVerdictArtifact(input, frozenTime);
var hash2 = ComputeCanonicalHash(verdict2);
// Assert
hash1.Should().Be(hash2, "Same input should produce same canonical hash");
hash1.Should().MatchRegex("^[0-9a-f]{64}$");
}
[Fact]
public void VerdictArtifact_DeterminismManifest_CanBeCreated()
{
// Arrange
var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z");
var input = CreateSampleVerdictInput();
var verdict = GenerateVerdictArtifact(input, frozenTime);
var verdictBytes = Encoding.UTF8.GetBytes(CanonJson.Serialize(verdict));
var artifactInfo = new ArtifactInfo
{
Type = "verdict-artifact",
Name = "test-delta-verdict",
Version = "1.0.0",
Format = "delta-verdict@1.0"
};
var toolchain = new ToolchainInfo
{
Platform = ".NET 10.0",
Components = new[]
{
new ComponentInfo { Name = "StellaOps.Scanner.SmartDiff", Version = "1.0.0" }
}
};
// Act - Create determinism manifest
var manifest = DeterminismManifestWriter.CreateManifest(
verdictBytes,
artifactInfo,
toolchain);
// Assert
manifest.SchemaVersion.Should().Be("1.0");
manifest.Artifact.Format.Should().Be("delta-verdict@1.0");
manifest.CanonicalHash.Algorithm.Should().Be("SHA-256");
manifest.CanonicalHash.Value.Should().MatchRegex("^[0-9a-f]{64}$");
}
[Fact]
public async Task VerdictArtifact_ParallelGeneration_ProducesDeterministicOutput()
{
// Arrange
var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z");
var input = CreateSampleVerdictInput();
// Act - Generate in parallel 20 times
var tasks = Enumerable.Range(0, 20)
.Select(_ => Task.Run(() => CanonJson.Serialize(GenerateVerdictArtifact(input, frozenTime))))
.ToArray();
var verdicts = await Task.WhenAll(tasks);
// Assert - All outputs should be identical
verdicts.Should().AllBe(verdicts[0]);
}
#endregion
#region Change Ordering Tests
[Fact]
public void VerdictArtifact_ChangesAreDeterministicallyOrdered()
{
// Arrange - Create input with changes in random order
var changes = new[]
{
CreateChange("CVE-2024-0003", "pkg:npm/c@1.0.0", "new"),
CreateChange("CVE-2024-0001", "pkg:npm/a@1.0.0", "resolved"),
CreateChange("CVE-2024-0002", "pkg:npm/b@1.0.0", "severity_changed")
};
var input = new VerdictInput
{
VerdictId = Guid.Parse("11111111-1111-1111-1111-111111111111"),
BaselineScanId = Guid.Parse("00000000-0000-0000-0000-000000000001"),
CurrentScanId = Guid.Parse("00000000-0000-0000-0000-000000000002"),
Changes = changes
};
var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z");
// Act
var verdict1 = GenerateVerdictArtifact(input, frozenTime);
var verdict2 = GenerateVerdictArtifact(input, frozenTime);
// Assert - Outputs should be identical
var json1 = CanonJson.Serialize(verdict1);
var json2 = CanonJson.Serialize(verdict2);
json1.Should().Be(json2);
// Verify changes are sorted by CVE ID, then by package URL
for (int i = 1; i < verdict1.Changes.Count; i++)
{
var cmp = string.CompareOrdinal(verdict1.Changes[i - 1].CveId, verdict1.Changes[i].CveId);
if (cmp == 0)
{
cmp = string.CompareOrdinal(verdict1.Changes[i - 1].PackageUrl, verdict1.Changes[i].PackageUrl);
}
cmp.Should().BeLessOrEqualTo(0, "Changes should be sorted by CVE ID, then package URL");
}
}
[Fact]
public void VerdictArtifact_ChangesWithSameCveAndPackage_SortedByChangeType()
{
// Arrange - Multiple changes for same CVE/package
var changes = new[]
{
CreateChange("CVE-2024-0001", "pkg:npm/test@1.0.0", "severity_changed"),
CreateChange("CVE-2024-0001", "pkg:npm/test@1.0.0", "status_changed"),
CreateChange("CVE-2024-0001", "pkg:npm/test@1.0.0", "epss_changed")
};
var input = new VerdictInput
{
VerdictId = Guid.Parse("22222222-2222-2222-2222-222222222222"),
BaselineScanId = Guid.Parse("00000000-0000-0000-0000-000000000001"),
CurrentScanId = Guid.Parse("00000000-0000-0000-0000-000000000002"),
Changes = changes
};
var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z");
// Act
var verdict1 = GenerateVerdictArtifact(input, frozenTime);
var verdict2 = GenerateVerdictArtifact(input, frozenTime);
// Assert
var json1 = CanonJson.Serialize(verdict1);
var json2 = CanonJson.Serialize(verdict2);
json1.Should().Be(json2);
}
#endregion
#region Change Type Tests
[Theory]
[InlineData("new")]
[InlineData("resolved")]
[InlineData("severity_changed")]
[InlineData("status_changed")]
[InlineData("epss_changed")]
[InlineData("reachability_changed")]
[InlineData("vex_status_changed")]
public void VerdictArtifact_ChangeTypeIsPreserved(string changeType)
{
// Arrange
var change = CreateChange("CVE-2024-0001", "pkg:npm/test@1.0.0", changeType);
var input = new VerdictInput
{
VerdictId = Guid.Parse("33333333-3333-3333-3333-333333333333"),
BaselineScanId = Guid.Parse("00000000-0000-0000-0000-000000000001"),
CurrentScanId = Guid.Parse("00000000-0000-0000-0000-000000000002"),
Changes = new[] { change }
};
var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z");
// Act
var verdict = GenerateVerdictArtifact(input, frozenTime);
// Assert
verdict.Changes[0].ChangeType.Should().Be(changeType);
}
#endregion
#region Proof Spine Tests
[Fact]
public void VerdictArtifact_ProofSpinesAreDeterministicallyOrdered()
{
// Arrange
var changes = new[]
{
CreateChange("CVE-2024-0001", "pkg:npm/a@1.0.0", "new") with
{
ProofSpine = new ProofSpine
{
SpineId = "spine-a",
Evidences = new[]
{
CreateProofEvidence("epss", 0.8),
CreateProofEvidence("reachability", 0.9),
CreateProofEvidence("vex", 1.0)
}
}
}
};
var input = new VerdictInput
{
VerdictId = Guid.Parse("44444444-4444-4444-4444-444444444444"),
BaselineScanId = Guid.Parse("00000000-0000-0000-0000-000000000001"),
CurrentScanId = Guid.Parse("00000000-0000-0000-0000-000000000002"),
Changes = changes
};
var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z");
// Act
var verdict1 = GenerateVerdictArtifact(input, frozenTime);
var verdict2 = GenerateVerdictArtifact(input, frozenTime);
// Assert
var json1 = CanonJson.Serialize(verdict1);
var json2 = CanonJson.Serialize(verdict2);
json1.Should().Be(json2);
// Verify evidences in proof spine are sorted
var evidences = verdict1.Changes[0].ProofSpine!.Evidences;
for (int i = 1; i < evidences.Count; i++)
{
string.CompareOrdinal(evidences[i - 1].EvidenceType, evidences[i].EvidenceType)
.Should().BeLessOrEqualTo(0, "Proof spine evidences should be sorted by type");
}
}
[Fact]
public void VerdictArtifact_ProofSpineHashIsStable()
{
// Arrange
var change = CreateChange("CVE-2024-0001", "pkg:npm/test@1.0.0", "new") with
{
ProofSpine = new ProofSpine
{
SpineId = "spine-test",
Evidences = new[]
{
CreateProofEvidence("epss", 0.5),
CreateProofEvidence("reachability", 0.75)
}
}
};
var input = new VerdictInput
{
VerdictId = Guid.Parse("55555555-5555-5555-5555-555555555555"),
BaselineScanId = Guid.Parse("00000000-0000-0000-0000-000000000001"),
CurrentScanId = Guid.Parse("00000000-0000-0000-0000-000000000002"),
Changes = new[] { change }
};
var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z");
// Act
var verdict1 = GenerateVerdictArtifact(input, frozenTime);
var verdict2 = GenerateVerdictArtifact(input, frozenTime);
// Assert
verdict1.Changes[0].ProofSpine!.SpineHash.Should().Be(verdict2.Changes[0].ProofSpine!.SpineHash);
verdict1.Changes[0].ProofSpine!.SpineHash.Should().MatchRegex("^[0-9a-f]{64}$");
}
#endregion
#region Summary Statistics Tests
[Fact]
public void VerdictArtifact_SummaryStatisticsAreDeterministic()
{
// Arrange
var changes = new[]
{
CreateChange("CVE-2024-0001", "pkg:npm/a@1.0.0", "new"),
CreateChange("CVE-2024-0002", "pkg:npm/b@1.0.0", "new"),
CreateChange("CVE-2024-0003", "pkg:npm/c@1.0.0", "resolved"),
CreateChange("CVE-2024-0004", "pkg:npm/d@1.0.0", "severity_changed")
};
var input = new VerdictInput
{
VerdictId = Guid.Parse("66666666-6666-6666-6666-666666666666"),
BaselineScanId = Guid.Parse("00000000-0000-0000-0000-000000000001"),
CurrentScanId = Guid.Parse("00000000-0000-0000-0000-000000000002"),
Changes = changes
};
var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z");
// Act
var verdict1 = GenerateVerdictArtifact(input, frozenTime);
var verdict2 = GenerateVerdictArtifact(input, frozenTime);
// Assert
verdict1.Summary.Should().NotBeNull();
verdict1.Summary.TotalChanges.Should().Be(verdict2.Summary.TotalChanges);
verdict1.Summary.NewFindings.Should().Be(verdict2.Summary.NewFindings);
verdict1.Summary.ResolvedFindings.Should().Be(verdict2.Summary.ResolvedFindings);
}
#endregion
#region Empty/Edge Case Tests
[Fact]
public void VerdictArtifact_NoChanges_ProducesDeterministicOutput()
{
// Arrange
var input = new VerdictInput
{
VerdictId = Guid.Parse("77777777-7777-7777-7777-777777777777"),
BaselineScanId = Guid.Parse("00000000-0000-0000-0000-000000000001"),
CurrentScanId = Guid.Parse("00000000-0000-0000-0000-000000000002"),
Changes = Array.Empty()
};
var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z");
// Act
var hash1 = ComputeCanonicalHash(GenerateVerdictArtifact(input, frozenTime));
var hash2 = ComputeCanonicalHash(GenerateVerdictArtifact(input, frozenTime));
// Assert
hash1.Should().Be(hash2);
}
[Fact]
public void VerdictArtifact_ManyChanges_ProducesDeterministicOutput()
{
// Arrange - Create 500 changes
var changes = Enumerable.Range(0, 500)
.Select(i => CreateChange(
$"CVE-2024-{i:D4}",
$"pkg:npm/package-{i}@1.0.0",
i % 3 == 0 ? "new" : i % 2 == 0 ? "resolved" : "severity_changed"))
.ToArray();
var input = new VerdictInput
{
VerdictId = Guid.Parse("88888888-8888-8888-8888-888888888888"),
BaselineScanId = Guid.Parse("00000000-0000-0000-0000-000000000001"),
CurrentScanId = Guid.Parse("00000000-0000-0000-0000-000000000002"),
Changes = changes
};
var frozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z");
// Act
var hash1 = ComputeCanonicalHash(GenerateVerdictArtifact(input, frozenTime));
var hash2 = ComputeCanonicalHash(GenerateVerdictArtifact(input, frozenTime));
// Assert
hash1.Should().Be(hash2);
}
#endregion
#region Helper Methods
private static VerdictInput CreateSampleVerdictInput()
{
return new VerdictInput
{
VerdictId = Guid.Parse("99999999-9999-9999-9999-999999999999"),
BaselineScanId = Guid.Parse("00000000-0000-0000-0000-000000000001"),
CurrentScanId = Guid.Parse("00000000-0000-0000-0000-000000000002"),
Changes = new[]
{
CreateChange("CVE-2024-1234", "pkg:npm/lodash@4.17.20", "new"),
CreateChange("CVE-2024-5678", "pkg:npm/axios@0.21.0", "resolved"),
CreateChange("CVE-2024-9012", "pkg:npm/express@4.17.1", "severity_changed")
}
};
}
private static VerdictChange CreateChange(string cveId, string packageUrl, string changeType)
{
return new VerdictChange
{
CveId = cveId,
PackageUrl = packageUrl,
ChangeType = changeType,
ProofSpine = null
};
}
private static ProofEvidence CreateProofEvidence(string evidenceType, double confidence)
{
return new ProofEvidence
{
EvidenceType = evidenceType,
Confidence = confidence,
Summary = $"{evidenceType} evidence"
};
}
private static VerdictArtifact GenerateVerdictArtifact(VerdictInput input, DateTimeOffset timestamp)
{
// Sort changes deterministically
var sortedChanges = input.Changes
.OrderBy(c => c.CveId, StringComparer.Ordinal)
.ThenBy(c => c.PackageUrl, StringComparer.Ordinal)
.ThenBy(c => c.ChangeType, StringComparer.Ordinal)
.Select(c => new VerdictChangeOutput
{
CveId = c.CveId,
PackageUrl = c.PackageUrl,
ChangeType = c.ChangeType,
ProofSpine = c.ProofSpine != null ? ProcessProofSpine(c.ProofSpine) : null
})
.ToList();
// Compute summary statistics
var summary = new VerdictSummary
{
TotalChanges = sortedChanges.Count,
NewFindings = sortedChanges.Count(c => c.ChangeType == "new"),
ResolvedFindings = sortedChanges.Count(c => c.ChangeType == "resolved"),
OtherChanges = sortedChanges.Count(c => c.ChangeType != "new" && c.ChangeType != "resolved")
};
return new VerdictArtifact
{
VerdictId = input.VerdictId,
BaselineScanId = input.BaselineScanId,
CurrentScanId = input.CurrentScanId,
Timestamp = timestamp,
Changes = sortedChanges,
Summary = summary
};
}
private static ProofSpineOutput ProcessProofSpine(ProofSpine spine)
{
var sortedEvidences = spine.Evidences
.OrderBy(e => e.EvidenceType, StringComparer.Ordinal)
.ToList();
// Compute spine hash from sorted evidences
var evidenceJson = CanonJson.Serialize(sortedEvidences);
var spineHash = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(evidenceJson));
return new ProofSpineOutput
{
SpineId = spine.SpineId,
Evidences = sortedEvidences,
SpineHash = spineHash
};
}
private static string ComputeCanonicalHash(VerdictArtifact artifact)
{
var json = CanonJson.Serialize(artifact);
return CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(json));
}
#endregion
#region DTOs
private sealed record VerdictInput
{
public required Guid VerdictId { get; init; }
public required Guid BaselineScanId { get; init; }
public required Guid CurrentScanId { get; init; }
public required VerdictChange[] Changes { get; init; }
}
private sealed record VerdictChange
{
public required string CveId { get; init; }
public required string PackageUrl { get; init; }
public required string ChangeType { get; init; }
public ProofSpine? ProofSpine { get; init; }
}
private sealed record ProofSpine
{
public required string SpineId { get; init; }
public required ProofEvidence[] Evidences { get; init; }
}
private sealed record ProofEvidence
{
public required string EvidenceType { get; init; }
public required double Confidence { get; init; }
public required string Summary { get; init; }
}
private sealed record VerdictArtifact
{
public required Guid VerdictId { get; init; }
public required Guid BaselineScanId { get; init; }
public required Guid CurrentScanId { get; init; }
public required DateTimeOffset Timestamp { get; init; }
public required IReadOnlyList Changes { get; init; }
public required VerdictSummary Summary { get; init; }
}
private sealed record VerdictChangeOutput
{
public required string CveId { get; init; }
public required string PackageUrl { get; init; }
public required string ChangeType { get; init; }
public ProofSpineOutput? ProofSpine { get; init; }
}
private sealed record ProofSpineOutput
{
public required string SpineId { get; init; }
public required IReadOnlyList Evidences { get; init; }
public required string SpineHash { get; init; }
}
private sealed record VerdictSummary
{
public required int TotalChanges { get; init; }
public required int NewFindings { get; init; }
public required int ResolvedFindings { get; init; }
public required int OtherChanges { get; init; }
}
#endregion
}