Files
git.stella-ops.org/src/__Tests/Integration/StellaOps.Integration.Determinism/BinaryEvidenceDeterminismTests.cs

715 lines
23 KiB
C#

// -----------------------------------------------------------------------------
// BinaryEvidenceDeterminismTests.cs
// Sprint: SPRINT_20251226_014_BINIDX
// Task: SCANINT-23 - Determinism tests for binary verdict reproducibility
// -----------------------------------------------------------------------------
using System.Collections.Immutable;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using FluentAssertions;
using StellaOps.Canonical.Json;
using StellaOps.Testing.Determinism;
using Xunit;
namespace StellaOps.Integration.Determinism;
/// <summary>
/// Determinism validation tests for binary vulnerability evidence.
/// Ensures identical binary inputs produce identical verdicts across:
/// - Binary identity extraction
/// - Vulnerability match computation
/// - Fix status determination
/// - Proof segment generation
/// - Multiple runs with frozen time
/// - Parallel execution
/// </summary>
public class BinaryEvidenceDeterminismTests
{
#region Binary Identity Determinism Tests
[Fact]
public void BinaryIdentity_WithIdenticalInput_ProducesDeterministicOutput()
{
// Arrange
var binaryData = CreateSampleBinaryData();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Extract identity multiple times
var identity1 = ExtractBinaryIdentity(binaryData, frozenTime);
var identity2 = ExtractBinaryIdentity(binaryData, frozenTime);
var identity3 = ExtractBinaryIdentity(binaryData, frozenTime);
// Assert - All outputs should be identical
identity1.Should().Be(identity2);
identity2.Should().Be(identity3);
}
[Fact]
public void BinaryIdentity_BuildId_IsStable()
{
// Arrange
var binaryData = CreateSampleBinaryData();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var identity1 = ExtractBinaryIdentity(binaryData, frozenTime);
var identity2 = ExtractBinaryIdentity(binaryData, frozenTime);
// Assert
identity1.BuildId.Should().Be(identity2.BuildId);
identity1.BuildId.Should().MatchRegex("^[0-9a-f]{40}$");
}
[Fact]
public void BinaryIdentity_BinaryKey_IsStable()
{
// Arrange
var binaryData = CreateSampleBinaryData();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var identity1 = ExtractBinaryIdentity(binaryData, frozenTime);
var identity2 = ExtractBinaryIdentity(binaryData, frozenTime);
// Assert
identity1.BinaryKey.Should().Be(identity2.BinaryKey);
}
[Fact]
public async Task BinaryIdentity_ParallelExtraction_ProducesDeterministicOutput()
{
// Arrange
var binaryData = CreateSampleBinaryData();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Extract in parallel 20 times
var tasks = Enumerable.Range(0, 20)
.Select(_ => Task.Run(() => ExtractBinaryIdentity(binaryData, frozenTime)))
.ToArray();
var identities = await Task.WhenAll(tasks);
// Assert - All outputs should be identical
identities.Should().OnlyContain(x => x == identities[0]);
}
#endregion
#region Vulnerability Match Determinism Tests
[Fact]
public void VulnMatch_WithIdenticalBinary_ProducesDeterministicMatches()
{
// Arrange
var identity = CreateSampleBinaryIdentity();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Look up matches multiple times
var matches1 = LookupVulnerabilities(identity, frozenTime);
var matches2 = LookupVulnerabilities(identity, frozenTime);
var matches3 = LookupVulnerabilities(identity, frozenTime);
// Assert - All results should be identical
var json1 = SerializeMatches(matches1);
var json2 = SerializeMatches(matches2);
var json3 = SerializeMatches(matches3);
json1.Should().Be(json2);
json2.Should().Be(json3);
}
[Fact]
public void VulnMatch_Ordering_IsDeterministic()
{
// Arrange
var identity = CreateSampleBinaryIdentityWithMultipleCves();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var matches1 = LookupVulnerabilities(identity, frozenTime);
var matches2 = LookupVulnerabilities(identity, frozenTime);
// Assert - CVEs should be in same order
var cves1 = matches1.Select(m => m.CveId).ToList();
var cves2 = matches2.Select(m => m.CveId).ToList();
cves1.Should().Equal(cves2);
}
[Fact]
public void VulnMatch_Confidence_IsDeterministic()
{
// Arrange
var identity = CreateSampleBinaryIdentity();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var matches1 = LookupVulnerabilities(identity, frozenTime);
var matches2 = LookupVulnerabilities(identity, frozenTime);
// Assert - Confidence scores should be identical
for (int i = 0; i < matches1.Length; i++)
{
matches1[i].Confidence.Should().Be(matches2[i].Confidence);
}
}
[Fact]
public void VulnMatch_CanonicalHash_IsStable()
{
// Arrange
var identity = CreateSampleBinaryIdentity();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var matches1 = LookupVulnerabilities(identity, frozenTime);
var json1 = SerializeMatches(matches1);
var hash1 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(json1));
var matches2 = LookupVulnerabilities(identity, frozenTime);
var json2 = SerializeMatches(matches2);
var hash2 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(json2));
// Assert
hash1.Should().Be(hash2);
hash1.Should().MatchRegex("^[0-9a-f]{64}$");
}
#endregion
#region Fix Status Determinism Tests
[Fact]
public void FixStatus_WithIdenticalInput_ProducesDeterministicOutput()
{
// Arrange
var input = CreateFixStatusInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var status1 = GetFixStatus(input, frozenTime);
var status2 = GetFixStatus(input, frozenTime);
var status3 = GetFixStatus(input, frozenTime);
// Assert
SerializeFixStatus(status1).Should().Be(SerializeFixStatus(status2));
SerializeFixStatus(status2).Should().Be(SerializeFixStatus(status3));
}
[Fact]
public void FixStatus_BackportDetection_IsDeterministic()
{
// Arrange
var input = CreateBackportedCveInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var status1 = GetFixStatus(input, frozenTime);
var status2 = GetFixStatus(input, frozenTime);
// Assert - Both should detect as fixed
status1.State.Should().Be("fixed");
status2.State.Should().Be("fixed");
status1.FixedVersion.Should().Be(status2.FixedVersion);
status1.Confidence.Should().Be(status2.Confidence);
}
[Fact]
public void FixStatus_Method_IsConsistent()
{
// Arrange
var input = CreateFixStatusInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var status = GetFixStatus(input, frozenTime);
// Assert - Method should be one of known values
status.Method.Should().BeOneOf("changelog", "patch_analysis", "advisory");
}
#endregion
#region Proof Segment Determinism Tests
[Fact]
public void ProofSegment_WithIdenticalEvidence_ProducesDeterministicOutput()
{
// Arrange
var evidence = CreateSampleBinaryEvidence();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
var deterministicId = GenerateDeterministicProofId(evidence, frozenTime);
// Act
var proof1 = CreateBinaryProofSegment(evidence, frozenTime, deterministicId);
var proof2 = CreateBinaryProofSegment(evidence, frozenTime, deterministicId);
var proof3 = CreateBinaryProofSegment(evidence, frozenTime, deterministicId);
// Assert
proof1.Should().Be(proof2);
proof2.Should().Be(proof3);
}
[Fact]
public void ProofSegment_CanonicalHash_IsStable()
{
// Arrange
var evidence = CreateSampleBinaryEvidence();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
var deterministicId = GenerateDeterministicProofId(evidence, frozenTime);
// Act
var proof1 = CreateBinaryProofSegment(evidence, frozenTime, deterministicId);
var hash1 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(proof1));
var proof2 = CreateBinaryProofSegment(evidence, frozenTime, deterministicId);
var hash2 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(proof2));
// Assert
hash1.Should().Be(hash2);
}
[Fact]
public void ProofSegment_PredicateType_IsConsistent()
{
// Arrange
var evidence = CreateSampleBinaryEvidence();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
var deterministicId = GenerateDeterministicProofId(evidence, frozenTime);
// Act
var proof = CreateBinaryProofSegment(evidence, frozenTime, deterministicId);
// Assert
proof.Should().Contain("\"predicateType\"");
proof.Should().Contain("https://stellaops.dev/predicates/binary-fingerprint-evidence@v1");
}
[Fact]
public async Task ProofSegment_ParallelGeneration_ProducesDeterministicOutput()
{
// Arrange
var evidence = CreateSampleBinaryEvidence();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
var deterministicId = GenerateDeterministicProofId(evidence, frozenTime);
// Act - Generate in parallel
var tasks = Enumerable.Range(0, 20)
.Select(_ => Task.Run(() => CreateBinaryProofSegment(evidence, frozenTime, deterministicId)))
.ToArray();
var proofs = await Task.WhenAll(tasks);
// Assert
proofs.Should().OnlyContain(x => x == proofs[0]);
}
#endregion
#region End-to-End Verdict Determinism Tests
[Fact]
public void FullBinaryVerdict_WithIdenticalInput_ProducesDeterministicOutput()
{
// Arrange
var scanInput = CreateSampleScanInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Process scan multiple times
var verdict1 = ProcessBinaryScan(scanInput, frozenTime);
var verdict2 = ProcessBinaryScan(scanInput, frozenTime);
var verdict3 = ProcessBinaryScan(scanInput, frozenTime);
// Assert
var json1 = SerializeVerdict(verdict1);
var json2 = SerializeVerdict(verdict2);
var json3 = SerializeVerdict(verdict3);
json1.Should().Be(json2);
json2.Should().Be(json3);
}
[Fact]
public void FullBinaryVerdict_CanonicalHash_IsStable()
{
// Arrange
var scanInput = CreateSampleScanInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var verdict1 = ProcessBinaryScan(scanInput, frozenTime);
var json1 = SerializeVerdict(verdict1);
var hash1 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(json1));
var verdict2 = ProcessBinaryScan(scanInput, frozenTime);
var json2 = SerializeVerdict(verdict2);
var hash2 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(json2));
// Assert
hash1.Should().Be(hash2);
}
[Fact]
public void FullBinaryVerdict_DeterminismManifest_CanBeCreated()
{
// Arrange
var scanInput = CreateSampleScanInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
var verdict = ProcessBinaryScan(scanInput, frozenTime);
var verdictBytes = Encoding.UTF8.GetBytes(SerializeVerdict(verdict));
var artifactInfo = new ArtifactInfo
{
Type = "binary-evidence",
Name = "binary-vulnerability-verdict",
Version = "1.0.0",
Format = "BinaryEvidence JSON"
};
var toolchain = new ToolchainInfo
{
Platform = ".NET 10.0",
Components = new[]
{
new ComponentInfo { Name = "StellaOps.BinaryIndex", Version = "1.0.0" }
}
};
// Act
var manifest = DeterminismManifestWriter.CreateManifest(
verdictBytes,
artifactInfo,
toolchain);
// Assert
manifest.SchemaVersion.Should().Be("1.0");
manifest.Artifact.Format.Should().Be("BinaryEvidence JSON");
manifest.CanonicalHash.Algorithm.Should().Be("SHA-256");
}
#endregion
#region Helper Methods
private static byte[] CreateSampleBinaryData()
{
// Simulated ELF binary data with Build-ID
var data = new byte[1024];
var random = new Random(42); // Deterministic seed
random.NextBytes(data);
// Add ELF magic header
data[0] = 0x7f;
data[1] = 0x45; // E
data[2] = 0x4c; // L
data[3] = 0x46; // F
return data;
}
private static BinaryIdentityResult ExtractBinaryIdentity(byte[] data, DateTimeOffset timestamp)
{
// Compute deterministic Build-ID from data
var buildId = ComputeDeterministicBuildId(data);
var fileSha256 = CanonJson.Sha256Hex(data);
return new BinaryIdentityResult
{
Format = "elf",
BuildId = buildId,
FileSha256 = $"sha256:{fileSha256}",
Architecture = "x86_64",
BinaryKey = $"test-binary:{buildId[..8]}"
};
}
private static BinaryIdentityResult CreateSampleBinaryIdentity()
{
return new BinaryIdentityResult
{
Format = "elf",
BuildId = "8d8f09a0d7e2c1b3a5f4e6d8c0b2a4e6f8d0c2b4",
FileSha256 = "sha256:abcd1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab",
Architecture = "x86_64",
BinaryKey = "openssl:1.1.1w-1"
};
}
private static BinaryIdentityResult CreateSampleBinaryIdentityWithMultipleCves()
{
return new BinaryIdentityResult
{
Format = "elf",
BuildId = "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0",
FileSha256 = "sha256:1111222233334444555566667777888899990000aaaabbbbccccddddeeeeffff",
Architecture = "x86_64",
BinaryKey = "curl:7.74.0-1"
};
}
private static VulnMatch[] LookupVulnerabilities(BinaryIdentityResult identity, DateTimeOffset timestamp)
{
// Deterministic vulnerability lookup based on binary key
var matches = new List<VulnMatch>();
if (identity.BinaryKey.Contains("openssl"))
{
matches.Add(new VulnMatch
{
CveId = "CVE-2023-5678",
Method = "buildid_catalog",
Confidence = 0.95m,
VulnerablePurl = "pkg:deb/debian/openssl@1.1.1n-0+deb11u4"
});
}
if (identity.BinaryKey.Contains("curl"))
{
matches.Add(new VulnMatch
{
CveId = "CVE-2023-38545",
Method = "buildid_catalog",
Confidence = 0.98m,
VulnerablePurl = "pkg:deb/debian/curl@7.74.0-1.3+deb11u5"
});
matches.Add(new VulnMatch
{
CveId = "CVE-2024-2398",
Method = "buildid_catalog",
Confidence = 0.96m,
VulnerablePurl = "pkg:deb/debian/curl@7.74.0-1.3+deb11u6"
});
}
// Sort by CVE ID for deterministic ordering
return matches.OrderBy(m => m.CveId, StringComparer.Ordinal).ToArray();
}
private static FixStatusInput CreateFixStatusInput()
{
return new FixStatusInput
{
Distro = "debian",
Release = "bookworm",
SourcePkg = "openssl",
CveId = "CVE-2023-5678"
};
}
private static FixStatusInput CreateBackportedCveInput()
{
return new FixStatusInput
{
Distro = "debian",
Release = "bookworm",
SourcePkg = "openssl",
CveId = "CVE-2023-4807"
};
}
private static FixStatusResult GetFixStatus(FixStatusInput input, DateTimeOffset timestamp)
{
// Deterministic fix status based on input
return new FixStatusResult
{
State = "fixed",
FixedVersion = "1.1.1w-1",
Method = "changelog",
Confidence = 0.98m
};
}
private static BinaryEvidence CreateSampleBinaryEvidence()
{
return new BinaryEvidence
{
Identity = CreateSampleBinaryIdentity(),
LayerDigest = "sha256:layer1abc123def456789012345678901234567890abcdef12345678901234",
Matches = LookupVulnerabilities(CreateSampleBinaryIdentity(), DateTimeOffset.UtcNow)
};
}
private static string GenerateDeterministicProofId(BinaryEvidence evidence, DateTimeOffset timestamp)
{
var seed = $"{evidence.Identity.BinaryKey}:{evidence.LayerDigest}:{timestamp:O}";
var hash = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(seed));
return $"proof:{hash[..32]}";
}
private static string CreateBinaryProofSegment(BinaryEvidence evidence, DateTimeOffset timestamp, string proofId)
{
var matchesJson = string.Join(",\n ", evidence.Matches.Select(m => $$"""
{
"cve_id": "{{m.CveId}}",
"method": "{{m.Method}}",
"confidence": {{m.Confidence.ToString("F2", System.Globalization.CultureInfo.InvariantCulture)}},
"vulnerable_purl": "{{m.VulnerablePurl}}"
}
"""));
return $$"""
{
"predicateType": "https://stellaops.dev/predicates/binary-fingerprint-evidence@v1",
"proofId": "{{proofId}}",
"createdAt": "{{timestamp:O}}",
"binaryIdentity": {
"format": "{{evidence.Identity.Format}}",
"buildId": "{{evidence.Identity.BuildId}}",
"fileSha256": "{{evidence.Identity.FileSha256}}",
"architecture": "{{evidence.Identity.Architecture}}",
"binaryKey": "{{evidence.Identity.BinaryKey}}"
},
"layerDigest": "{{evidence.LayerDigest}}",
"matches": [
{{matchesJson}}
]
}
""";
}
private static ScanInput CreateSampleScanInput()
{
return new ScanInput
{
ImageDigest = "sha256:9f92a8c39f8d4f7bb1a60f2be650b3019b9a1bb50d2da839efa9bf2a278a0071",
Distro = "debian",
Release = "bookworm",
Binaries = new[]
{
CreateSampleBinaryData()
}
};
}
private static BinaryVerdict ProcessBinaryScan(ScanInput input, DateTimeOffset timestamp)
{
var binaries = new List<BinaryEvidence>();
foreach (var binaryData in input.Binaries)
{
var identity = ExtractBinaryIdentity(binaryData, timestamp);
var matches = LookupVulnerabilities(identity, timestamp);
binaries.Add(new BinaryEvidence
{
Identity = identity,
LayerDigest = "sha256:layer1",
Matches = matches
});
}
return new BinaryVerdict
{
ScanId = GenerateScanId(input, timestamp),
ImageDigest = input.ImageDigest,
ScannedAt = timestamp,
Binaries = binaries.ToArray()
};
}
private static string GenerateScanId(ScanInput input, DateTimeOffset timestamp)
{
var seed = $"{input.ImageDigest}:{timestamp:O}";
var hash = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(seed));
return $"scan-{hash[..16]}";
}
private static string ComputeDeterministicBuildId(byte[] data)
{
using var sha1 = SHA1.Create();
var hash = sha1.ComputeHash(data);
return Convert.ToHexString(hash).ToLowerInvariant();
}
private static string SerializeMatches(VulnMatch[] matches)
{
return JsonSerializer.Serialize(matches, new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
WriteIndented = false
});
}
private static string SerializeFixStatus(FixStatusResult status)
{
return JsonSerializer.Serialize(status, new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
WriteIndented = false
});
}
private static string SerializeVerdict(BinaryVerdict verdict)
{
return JsonSerializer.Serialize(verdict, new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
WriteIndented = false
});
}
#endregion
#region DTOs
private sealed record BinaryIdentityResult
{
public required string Format { get; init; }
public required string BuildId { get; init; }
public required string FileSha256 { get; init; }
public required string Architecture { get; init; }
public required string BinaryKey { get; init; }
}
private sealed record VulnMatch
{
public required string CveId { get; init; }
public required string Method { get; init; }
public required decimal Confidence { get; init; }
public required string VulnerablePurl { get; init; }
}
private sealed record FixStatusInput
{
public required string Distro { get; init; }
public required string Release { get; init; }
public required string SourcePkg { get; init; }
public required string CveId { get; init; }
}
private sealed record FixStatusResult
{
public required string State { get; init; }
public required string FixedVersion { get; init; }
public required string Method { get; init; }
public required decimal Confidence { get; init; }
}
private sealed record BinaryEvidence
{
public required BinaryIdentityResult Identity { get; init; }
public required string LayerDigest { get; init; }
public required VulnMatch[] Matches { get; init; }
}
private sealed record ScanInput
{
public required string ImageDigest { get; init; }
public required string Distro { get; init; }
public required string Release { get; init; }
public required byte[][] Binaries { get; init; }
}
private sealed record BinaryVerdict
{
public required string ScanId { get; init; }
public required string ImageDigest { get; init; }
public required DateTimeOffset ScannedAt { get; init; }
public required BinaryEvidence[] Binaries { get; init; }
}
#endregion
}