// ----------------------------------------------------------------------------- // BinaryEvidenceDeterminismTests.cs // Sprint: SPRINT_20251226_014_BINIDX // Task: SCANINT-23 - Determinism tests for binary verdict reproducibility // ----------------------------------------------------------------------------- using System.Collections.Immutable; using System.Security.Cryptography; using System.Text; using System.Text.Json; using FluentAssertions; using StellaOps.Canonical.Json; using StellaOps.Testing.Determinism; using Xunit; namespace StellaOps.Integration.Determinism; /// /// Determinism validation tests for binary vulnerability evidence. /// Ensures identical binary inputs produce identical verdicts across: /// - Binary identity extraction /// - Vulnerability match computation /// - Fix status determination /// - Proof segment generation /// - Multiple runs with frozen time /// - Parallel execution /// public class BinaryEvidenceDeterminismTests { #region Binary Identity Determinism Tests [Fact] public void BinaryIdentity_WithIdenticalInput_ProducesDeterministicOutput() { // Arrange var binaryData = CreateSampleBinaryData(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act - Extract identity multiple times var identity1 = ExtractBinaryIdentity(binaryData, frozenTime); var identity2 = ExtractBinaryIdentity(binaryData, frozenTime); var identity3 = ExtractBinaryIdentity(binaryData, frozenTime); // Assert - All outputs should be identical identity1.Should().Be(identity2); identity2.Should().Be(identity3); } [Fact] public void BinaryIdentity_BuildId_IsStable() { // Arrange var binaryData = CreateSampleBinaryData(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act var identity1 = ExtractBinaryIdentity(binaryData, frozenTime); var identity2 = ExtractBinaryIdentity(binaryData, frozenTime); // Assert identity1.BuildId.Should().Be(identity2.BuildId); identity1.BuildId.Should().MatchRegex("^[0-9a-f]{40}$"); } [Fact] public void BinaryIdentity_BinaryKey_IsStable() { // Arrange var binaryData = CreateSampleBinaryData(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act var identity1 = ExtractBinaryIdentity(binaryData, frozenTime); var identity2 = ExtractBinaryIdentity(binaryData, frozenTime); // Assert identity1.BinaryKey.Should().Be(identity2.BinaryKey); } [Fact] public async Task BinaryIdentity_ParallelExtraction_ProducesDeterministicOutput() { // Arrange var binaryData = CreateSampleBinaryData(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act - Extract in parallel 20 times var tasks = Enumerable.Range(0, 20) .Select(_ => Task.Run(() => ExtractBinaryIdentity(binaryData, frozenTime))) .ToArray(); var identities = await Task.WhenAll(tasks); // Assert - All outputs should be identical identities.Should().OnlyContain(x => x == identities[0]); } #endregion #region Vulnerability Match Determinism Tests [Fact] public void VulnMatch_WithIdenticalBinary_ProducesDeterministicMatches() { // Arrange var identity = CreateSampleBinaryIdentity(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act - Look up matches multiple times var matches1 = LookupVulnerabilities(identity, frozenTime); var matches2 = LookupVulnerabilities(identity, frozenTime); var matches3 = LookupVulnerabilities(identity, frozenTime); // Assert - All results should be identical var json1 = SerializeMatches(matches1); var json2 = SerializeMatches(matches2); var json3 = SerializeMatches(matches3); json1.Should().Be(json2); json2.Should().Be(json3); } [Fact] public void VulnMatch_Ordering_IsDeterministic() { // Arrange var identity = CreateSampleBinaryIdentityWithMultipleCves(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act var matches1 = LookupVulnerabilities(identity, frozenTime); var matches2 = LookupVulnerabilities(identity, frozenTime); // Assert - CVEs should be in same order var cves1 = matches1.Select(m => m.CveId).ToList(); var cves2 = matches2.Select(m => m.CveId).ToList(); cves1.Should().Equal(cves2); } [Fact] public void VulnMatch_Confidence_IsDeterministic() { // Arrange var identity = CreateSampleBinaryIdentity(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act var matches1 = LookupVulnerabilities(identity, frozenTime); var matches2 = LookupVulnerabilities(identity, frozenTime); // Assert - Confidence scores should be identical for (int i = 0; i < matches1.Length; i++) { matches1[i].Confidence.Should().Be(matches2[i].Confidence); } } [Fact] public void VulnMatch_CanonicalHash_IsStable() { // Arrange var identity = CreateSampleBinaryIdentity(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act var matches1 = LookupVulnerabilities(identity, frozenTime); var json1 = SerializeMatches(matches1); var hash1 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(json1)); var matches2 = LookupVulnerabilities(identity, frozenTime); var json2 = SerializeMatches(matches2); var hash2 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(json2)); // Assert hash1.Should().Be(hash2); hash1.Should().MatchRegex("^[0-9a-f]{64}$"); } #endregion #region Fix Status Determinism Tests [Fact] public void FixStatus_WithIdenticalInput_ProducesDeterministicOutput() { // Arrange var input = CreateFixStatusInput(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act var status1 = GetFixStatus(input, frozenTime); var status2 = GetFixStatus(input, frozenTime); var status3 = GetFixStatus(input, frozenTime); // Assert SerializeFixStatus(status1).Should().Be(SerializeFixStatus(status2)); SerializeFixStatus(status2).Should().Be(SerializeFixStatus(status3)); } [Fact] public void FixStatus_BackportDetection_IsDeterministic() { // Arrange var input = CreateBackportedCveInput(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act var status1 = GetFixStatus(input, frozenTime); var status2 = GetFixStatus(input, frozenTime); // Assert - Both should detect as fixed status1.State.Should().Be("fixed"); status2.State.Should().Be("fixed"); status1.FixedVersion.Should().Be(status2.FixedVersion); status1.Confidence.Should().Be(status2.Confidence); } [Fact] public void FixStatus_Method_IsConsistent() { // Arrange var input = CreateFixStatusInput(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act var status = GetFixStatus(input, frozenTime); // Assert - Method should be one of known values status.Method.Should().BeOneOf("changelog", "patch_analysis", "advisory"); } #endregion #region Proof Segment Determinism Tests [Fact] public void ProofSegment_WithIdenticalEvidence_ProducesDeterministicOutput() { // Arrange var evidence = CreateSampleBinaryEvidence(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); var deterministicId = GenerateDeterministicProofId(evidence, frozenTime); // Act var proof1 = CreateBinaryProofSegment(evidence, frozenTime, deterministicId); var proof2 = CreateBinaryProofSegment(evidence, frozenTime, deterministicId); var proof3 = CreateBinaryProofSegment(evidence, frozenTime, deterministicId); // Assert proof1.Should().Be(proof2); proof2.Should().Be(proof3); } [Fact] public void ProofSegment_CanonicalHash_IsStable() { // Arrange var evidence = CreateSampleBinaryEvidence(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); var deterministicId = GenerateDeterministicProofId(evidence, frozenTime); // Act var proof1 = CreateBinaryProofSegment(evidence, frozenTime, deterministicId); var hash1 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(proof1)); var proof2 = CreateBinaryProofSegment(evidence, frozenTime, deterministicId); var hash2 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(proof2)); // Assert hash1.Should().Be(hash2); } [Fact] public void ProofSegment_PredicateType_IsConsistent() { // Arrange var evidence = CreateSampleBinaryEvidence(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); var deterministicId = GenerateDeterministicProofId(evidence, frozenTime); // Act var proof = CreateBinaryProofSegment(evidence, frozenTime, deterministicId); // Assert proof.Should().Contain("\"predicateType\""); proof.Should().Contain("https://stellaops.dev/predicates/binary-fingerprint-evidence@v1"); } [Fact] public async Task ProofSegment_ParallelGeneration_ProducesDeterministicOutput() { // Arrange var evidence = CreateSampleBinaryEvidence(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); var deterministicId = GenerateDeterministicProofId(evidence, frozenTime); // Act - Generate in parallel var tasks = Enumerable.Range(0, 20) .Select(_ => Task.Run(() => CreateBinaryProofSegment(evidence, frozenTime, deterministicId))) .ToArray(); var proofs = await Task.WhenAll(tasks); // Assert proofs.Should().OnlyContain(x => x == proofs[0]); } #endregion #region End-to-End Verdict Determinism Tests [Fact] public void FullBinaryVerdict_WithIdenticalInput_ProducesDeterministicOutput() { // Arrange var scanInput = CreateSampleScanInput(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act - Process scan multiple times var verdict1 = ProcessBinaryScan(scanInput, frozenTime); var verdict2 = ProcessBinaryScan(scanInput, frozenTime); var verdict3 = ProcessBinaryScan(scanInput, frozenTime); // Assert var json1 = SerializeVerdict(verdict1); var json2 = SerializeVerdict(verdict2); var json3 = SerializeVerdict(verdict3); json1.Should().Be(json2); json2.Should().Be(json3); } [Fact] public void FullBinaryVerdict_CanonicalHash_IsStable() { // Arrange var scanInput = CreateSampleScanInput(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act var verdict1 = ProcessBinaryScan(scanInput, frozenTime); var json1 = SerializeVerdict(verdict1); var hash1 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(json1)); var verdict2 = ProcessBinaryScan(scanInput, frozenTime); var json2 = SerializeVerdict(verdict2); var hash2 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(json2)); // Assert hash1.Should().Be(hash2); } [Fact] public void FullBinaryVerdict_DeterminismManifest_CanBeCreated() { // Arrange var scanInput = CreateSampleScanInput(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); var verdict = ProcessBinaryScan(scanInput, frozenTime); var verdictBytes = Encoding.UTF8.GetBytes(SerializeVerdict(verdict)); var artifactInfo = new ArtifactInfo { Type = "binary-evidence", Name = "binary-vulnerability-verdict", Version = "1.0.0", Format = "BinaryEvidence JSON" }; var toolchain = new ToolchainInfo { Platform = ".NET 10.0", Components = new[] { new ComponentInfo { Name = "StellaOps.BinaryIndex", Version = "1.0.0" } } }; // Act var manifest = DeterminismManifestWriter.CreateManifest( verdictBytes, artifactInfo, toolchain); // Assert manifest.SchemaVersion.Should().Be("1.0"); manifest.Artifact.Format.Should().Be("BinaryEvidence JSON"); manifest.CanonicalHash.Algorithm.Should().Be("SHA-256"); } #endregion #region Helper Methods private static byte[] CreateSampleBinaryData() { // Simulated ELF binary data with Build-ID var data = new byte[1024]; var random = new Random(42); // Deterministic seed random.NextBytes(data); // Add ELF magic header data[0] = 0x7f; data[1] = 0x45; // E data[2] = 0x4c; // L data[3] = 0x46; // F return data; } private static BinaryIdentityResult ExtractBinaryIdentity(byte[] data, DateTimeOffset timestamp) { // Compute deterministic Build-ID from data var buildId = ComputeDeterministicBuildId(data); var fileSha256 = CanonJson.Sha256Hex(data); return new BinaryIdentityResult { Format = "elf", BuildId = buildId, FileSha256 = $"sha256:{fileSha256}", Architecture = "x86_64", BinaryKey = $"test-binary:{buildId[..8]}" }; } private static BinaryIdentityResult CreateSampleBinaryIdentity() { return new BinaryIdentityResult { Format = "elf", BuildId = "8d8f09a0d7e2c1b3a5f4e6d8c0b2a4e6f8d0c2b4", FileSha256 = "sha256:abcd1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab", Architecture = "x86_64", BinaryKey = "openssl:1.1.1w-1" }; } private static BinaryIdentityResult CreateSampleBinaryIdentityWithMultipleCves() { return new BinaryIdentityResult { Format = "elf", BuildId = "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0", FileSha256 = "sha256:1111222233334444555566667777888899990000aaaabbbbccccddddeeeeffff", Architecture = "x86_64", BinaryKey = "curl:7.74.0-1" }; } private static VulnMatch[] LookupVulnerabilities(BinaryIdentityResult identity, DateTimeOffset timestamp) { // Deterministic vulnerability lookup based on binary key var matches = new List(); if (identity.BinaryKey.Contains("openssl")) { matches.Add(new VulnMatch { CveId = "CVE-2023-5678", Method = "buildid_catalog", Confidence = 0.95m, VulnerablePurl = "pkg:deb/debian/openssl@1.1.1n-0+deb11u4" }); } if (identity.BinaryKey.Contains("curl")) { matches.Add(new VulnMatch { CveId = "CVE-2023-38545", Method = "buildid_catalog", Confidence = 0.98m, VulnerablePurl = "pkg:deb/debian/curl@7.74.0-1.3+deb11u5" }); matches.Add(new VulnMatch { CveId = "CVE-2024-2398", Method = "buildid_catalog", Confidence = 0.96m, VulnerablePurl = "pkg:deb/debian/curl@7.74.0-1.3+deb11u6" }); } // Sort by CVE ID for deterministic ordering return matches.OrderBy(m => m.CveId, StringComparer.Ordinal).ToArray(); } private static FixStatusInput CreateFixStatusInput() { return new FixStatusInput { Distro = "debian", Release = "bookworm", SourcePkg = "openssl", CveId = "CVE-2023-5678" }; } private static FixStatusInput CreateBackportedCveInput() { return new FixStatusInput { Distro = "debian", Release = "bookworm", SourcePkg = "openssl", CveId = "CVE-2023-4807" }; } private static FixStatusResult GetFixStatus(FixStatusInput input, DateTimeOffset timestamp) { // Deterministic fix status based on input return new FixStatusResult { State = "fixed", FixedVersion = "1.1.1w-1", Method = "changelog", Confidence = 0.98m }; } private static BinaryEvidence CreateSampleBinaryEvidence() { return new BinaryEvidence { Identity = CreateSampleBinaryIdentity(), LayerDigest = "sha256:layer1abc123def456789012345678901234567890abcdef12345678901234", Matches = LookupVulnerabilities(CreateSampleBinaryIdentity(), DateTimeOffset.UtcNow) }; } private static string GenerateDeterministicProofId(BinaryEvidence evidence, DateTimeOffset timestamp) { var seed = $"{evidence.Identity.BinaryKey}:{evidence.LayerDigest}:{timestamp:O}"; var hash = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(seed)); return $"proof:{hash[..32]}"; } private static string CreateBinaryProofSegment(BinaryEvidence evidence, DateTimeOffset timestamp, string proofId) { var matchesJson = string.Join(",\n ", evidence.Matches.Select(m => $$""" { "cve_id": "{{m.CveId}}", "method": "{{m.Method}}", "confidence": {{m.Confidence.ToString("F2", System.Globalization.CultureInfo.InvariantCulture)}}, "vulnerable_purl": "{{m.VulnerablePurl}}" } """)); return $$""" { "predicateType": "https://stellaops.dev/predicates/binary-fingerprint-evidence@v1", "proofId": "{{proofId}}", "createdAt": "{{timestamp:O}}", "binaryIdentity": { "format": "{{evidence.Identity.Format}}", "buildId": "{{evidence.Identity.BuildId}}", "fileSha256": "{{evidence.Identity.FileSha256}}", "architecture": "{{evidence.Identity.Architecture}}", "binaryKey": "{{evidence.Identity.BinaryKey}}" }, "layerDigest": "{{evidence.LayerDigest}}", "matches": [ {{matchesJson}} ] } """; } private static ScanInput CreateSampleScanInput() { return new ScanInput { ImageDigest = "sha256:9f92a8c39f8d4f7bb1a60f2be650b3019b9a1bb50d2da839efa9bf2a278a0071", Distro = "debian", Release = "bookworm", Binaries = new[] { CreateSampleBinaryData() } }; } private static BinaryVerdict ProcessBinaryScan(ScanInput input, DateTimeOffset timestamp) { var binaries = new List(); foreach (var binaryData in input.Binaries) { var identity = ExtractBinaryIdentity(binaryData, timestamp); var matches = LookupVulnerabilities(identity, timestamp); binaries.Add(new BinaryEvidence { Identity = identity, LayerDigest = "sha256:layer1", Matches = matches }); } return new BinaryVerdict { ScanId = GenerateScanId(input, timestamp), ImageDigest = input.ImageDigest, ScannedAt = timestamp, Binaries = binaries.ToArray() }; } private static string GenerateScanId(ScanInput input, DateTimeOffset timestamp) { var seed = $"{input.ImageDigest}:{timestamp:O}"; var hash = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(seed)); return $"scan-{hash[..16]}"; } private static string ComputeDeterministicBuildId(byte[] data) { using var sha1 = SHA1.Create(); var hash = sha1.ComputeHash(data); return Convert.ToHexString(hash).ToLowerInvariant(); } private static string SerializeMatches(VulnMatch[] matches) { return JsonSerializer.Serialize(matches, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, WriteIndented = false }); } private static string SerializeFixStatus(FixStatusResult status) { return JsonSerializer.Serialize(status, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, WriteIndented = false }); } private static string SerializeVerdict(BinaryVerdict verdict) { return JsonSerializer.Serialize(verdict, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, WriteIndented = false }); } #endregion #region DTOs private sealed record BinaryIdentityResult { public required string Format { get; init; } public required string BuildId { get; init; } public required string FileSha256 { get; init; } public required string Architecture { get; init; } public required string BinaryKey { get; init; } } private sealed record VulnMatch { public required string CveId { get; init; } public required string Method { get; init; } public required decimal Confidence { get; init; } public required string VulnerablePurl { get; init; } } private sealed record FixStatusInput { public required string Distro { get; init; } public required string Release { get; init; } public required string SourcePkg { get; init; } public required string CveId { get; init; } } private sealed record FixStatusResult { public required string State { get; init; } public required string FixedVersion { get; init; } public required string Method { get; init; } public required decimal Confidence { get; init; } } private sealed record BinaryEvidence { public required BinaryIdentityResult Identity { get; init; } public required string LayerDigest { get; init; } public required VulnMatch[] Matches { get; init; } } private sealed record ScanInput { public required string ImageDigest { get; init; } public required string Distro { get; init; } public required string Release { get; init; } public required byte[][] Binaries { get; init; } } private sealed record BinaryVerdict { public required string ScanId { get; init; } public required string ImageDigest { get; init; } public required DateTimeOffset ScannedAt { get; init; } public required BinaryEvidence[] Binaries { get; init; } } #endregion }