5590a99a1a
- Created `StellaOps.TestKit.Tests` project for unit tests related to determinism. - Implemented `DeterminismManifestTests` to validate deterministic output for canonical bytes and strings, file read/write operations, and error handling for invalid schema versions. - Added `SbomDeterminismTests` to ensure identical inputs produce consistent SBOMs across SPDX 3.0.1 and CycloneDX 1.6/1.7 formats, including parallel execution tests. - Updated project references in `StellaOps.Integration.Determinism` to include the new determinism testing library.
561 lines
20 KiB
C#
561 lines
20 KiB
C#
// -----------------------------------------------------------------------------
|
|
// EvidenceBundleDeterminismTests.cs
|
|
// Sprint: SPRINT_5100_0007_0003 - Epic B (Determinism Gate)
|
|
// Task: T6 - Evidence Bundle Determinism (DSSE envelopes, in-toto attestations)
|
|
// Description: Tests to validate evidence bundle generation determinism
|
|
// -----------------------------------------------------------------------------
|
|
|
|
using System.Security.Cryptography;
|
|
using System.Text;
|
|
using FluentAssertions;
|
|
using StellaOps.Canonical.Json;
|
|
using StellaOps.Testing.Determinism;
|
|
using Xunit;
|
|
|
|
namespace StellaOps.Integration.Determinism;
|
|
|
|
/// <summary>
|
|
/// Determinism validation tests for evidence bundle generation.
|
|
/// Ensures identical inputs produce identical bundles across:
|
|
/// - Evidence bundle creation
|
|
/// - DSSE envelope wrapping
|
|
/// - in-toto attestation generation
|
|
/// - Multiple runs with frozen time
|
|
/// - Parallel execution
|
|
/// </summary>
|
|
public class EvidenceBundleDeterminismTests
|
|
{
|
|
#region Evidence Bundle Determinism Tests
|
|
|
|
[Fact]
|
|
public void EvidenceBundle_WithIdenticalInput_ProducesDeterministicOutput()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleEvidenceInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
var deterministicBundleId = GenerateDeterministicBundleId(input, frozenTime);
|
|
|
|
// Act - Generate bundle multiple times
|
|
var bundle1 = CreateEvidenceBundle(input, frozenTime, deterministicBundleId);
|
|
var bundle2 = CreateEvidenceBundle(input, frozenTime, deterministicBundleId);
|
|
var bundle3 = CreateEvidenceBundle(input, frozenTime, deterministicBundleId);
|
|
|
|
// Assert - All outputs should be identical
|
|
bundle1.Should().Be(bundle2);
|
|
bundle2.Should().Be(bundle3);
|
|
}
|
|
|
|
[Fact]
|
|
public void EvidenceBundle_CanonicalHash_IsStable()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleEvidenceInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
var deterministicBundleId = GenerateDeterministicBundleId(input, frozenTime);
|
|
|
|
// Act - Generate bundle and compute canonical hash twice
|
|
var bundle1 = CreateEvidenceBundle(input, frozenTime, deterministicBundleId);
|
|
var hash1 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(bundle1));
|
|
|
|
var bundle2 = CreateEvidenceBundle(input, frozenTime, deterministicBundleId);
|
|
var hash2 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(bundle2));
|
|
|
|
// Assert
|
|
hash1.Should().Be(hash2, "Same input should produce same canonical hash");
|
|
hash1.Should().MatchRegex("^[0-9a-f]{64}$");
|
|
}
|
|
|
|
[Fact]
|
|
public void EvidenceBundle_DeterminismManifest_CanBeCreated()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleEvidenceInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
var deterministicBundleId = GenerateDeterministicBundleId(input, frozenTime);
|
|
var bundle = CreateEvidenceBundle(input, frozenTime, deterministicBundleId);
|
|
var bundleBytes = Encoding.UTF8.GetBytes(bundle);
|
|
|
|
var artifactInfo = new ArtifactInfo
|
|
{
|
|
Type = "evidence-bundle",
|
|
Name = "test-finding-evidence",
|
|
Version = "1.0.0",
|
|
Format = "EvidenceBundle JSON"
|
|
};
|
|
|
|
var toolchain = new ToolchainInfo
|
|
{
|
|
Platform = ".NET 10.0",
|
|
Components = new[]
|
|
{
|
|
new ComponentInfo { Name = "StellaOps.Evidence.Bundle", Version = "1.0.0" }
|
|
}
|
|
};
|
|
|
|
// Act - Create determinism manifest
|
|
var manifest = DeterminismManifestWriter.CreateManifest(
|
|
bundleBytes,
|
|
artifactInfo,
|
|
toolchain);
|
|
|
|
// Assert
|
|
manifest.SchemaVersion.Should().Be("1.0");
|
|
manifest.Artifact.Format.Should().Be("EvidenceBundle JSON");
|
|
manifest.CanonicalHash.Algorithm.Should().Be("SHA-256");
|
|
manifest.CanonicalHash.Value.Should().MatchRegex("^[0-9a-f]{64}$");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task EvidenceBundle_ParallelGeneration_ProducesDeterministicOutput()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleEvidenceInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
var deterministicBundleId = GenerateDeterministicBundleId(input, frozenTime);
|
|
|
|
// Act - Generate in parallel 20 times
|
|
var tasks = Enumerable.Range(0, 20)
|
|
.Select(_ => Task.Run(() => CreateEvidenceBundle(input, frozenTime, deterministicBundleId)))
|
|
.ToArray();
|
|
|
|
var bundles = await Task.WhenAll(tasks);
|
|
|
|
// Assert - All outputs should be identical
|
|
bundles.Should().AllBe(bundles[0]);
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region DSSE Envelope Determinism Tests
|
|
|
|
[Fact]
|
|
public void DsseEnvelope_WithIdenticalPayload_ProducesDeterministicOutput()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleEvidenceInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
var deterministicBundleId = GenerateDeterministicBundleId(input, frozenTime);
|
|
var bundle = CreateEvidenceBundle(input, frozenTime, deterministicBundleId);
|
|
|
|
// Act - Wrap in DSSE envelope multiple times
|
|
var envelope1 = CreateDsseEnvelope(bundle, frozenTime);
|
|
var envelope2 = CreateDsseEnvelope(bundle, frozenTime);
|
|
var envelope3 = CreateDsseEnvelope(bundle, frozenTime);
|
|
|
|
// Assert - Payloads should be identical (signatures depend on key)
|
|
var payload1 = ExtractDssePayload(envelope1);
|
|
var payload2 = ExtractDssePayload(envelope2);
|
|
var payload3 = ExtractDssePayload(envelope3);
|
|
|
|
payload1.Should().Be(payload2);
|
|
payload2.Should().Be(payload3);
|
|
}
|
|
|
|
[Fact]
|
|
public void DsseEnvelope_PayloadHash_IsStable()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleEvidenceInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
var deterministicBundleId = GenerateDeterministicBundleId(input, frozenTime);
|
|
var bundle = CreateEvidenceBundle(input, frozenTime, deterministicBundleId);
|
|
|
|
// Act
|
|
var envelope1 = CreateDsseEnvelope(bundle, frozenTime);
|
|
var payload1 = ExtractDssePayload(envelope1);
|
|
var hash1 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(payload1));
|
|
|
|
var envelope2 = CreateDsseEnvelope(bundle, frozenTime);
|
|
var payload2 = ExtractDssePayload(envelope2);
|
|
var hash2 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(payload2));
|
|
|
|
// Assert
|
|
hash1.Should().Be(hash2);
|
|
hash1.Should().MatchRegex("^[0-9a-f]{64}$");
|
|
}
|
|
|
|
[Fact]
|
|
public void DsseEnvelope_PayloadType_IsConsistent()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleEvidenceInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
var deterministicBundleId = GenerateDeterministicBundleId(input, frozenTime);
|
|
var bundle = CreateEvidenceBundle(input, frozenTime, deterministicBundleId);
|
|
|
|
// Act
|
|
var envelope = CreateDsseEnvelope(bundle, frozenTime);
|
|
|
|
// Assert
|
|
envelope.Should().Contain("\"payloadType\"");
|
|
envelope.Should().Contain("application/vnd.stellaops.evidence+json");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region in-toto Attestation Determinism Tests
|
|
|
|
[Fact]
|
|
public void InTotoAttestation_WithIdenticalSubject_ProducesDeterministicOutput()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleEvidenceInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
var deterministicBundleId = GenerateDeterministicBundleId(input, frozenTime);
|
|
|
|
// Act - Generate attestation multiple times
|
|
var attestation1 = CreateInTotoAttestation(input, frozenTime, deterministicBundleId);
|
|
var attestation2 = CreateInTotoAttestation(input, frozenTime, deterministicBundleId);
|
|
var attestation3 = CreateInTotoAttestation(input, frozenTime, deterministicBundleId);
|
|
|
|
// Assert - All outputs should be identical
|
|
attestation1.Should().Be(attestation2);
|
|
attestation2.Should().Be(attestation3);
|
|
}
|
|
|
|
[Fact]
|
|
public void InTotoAttestation_CanonicalHash_IsStable()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleEvidenceInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
var deterministicBundleId = GenerateDeterministicBundleId(input, frozenTime);
|
|
|
|
// Act
|
|
var attestation1 = CreateInTotoAttestation(input, frozenTime, deterministicBundleId);
|
|
var hash1 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(attestation1));
|
|
|
|
var attestation2 = CreateInTotoAttestation(input, frozenTime, deterministicBundleId);
|
|
var hash2 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(attestation2));
|
|
|
|
// Assert
|
|
hash1.Should().Be(hash2);
|
|
}
|
|
|
|
[Fact]
|
|
public void InTotoAttestation_SubjectOrdering_IsDeterministic()
|
|
{
|
|
// Arrange - Multiple subjects
|
|
var input = CreateMultiSubjectEvidenceInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
var deterministicBundleId = GenerateDeterministicBundleId(input, frozenTime);
|
|
|
|
// Act
|
|
var attestation1 = CreateInTotoAttestation(input, frozenTime, deterministicBundleId);
|
|
var attestation2 = CreateInTotoAttestation(input, frozenTime, deterministicBundleId);
|
|
|
|
// Assert - Subject order should be deterministic
|
|
attestation1.Should().Be(attestation2);
|
|
}
|
|
|
|
[Fact]
|
|
public void InTotoAttestation_PredicateType_IsConsistent()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleEvidenceInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
var deterministicBundleId = GenerateDeterministicBundleId(input, frozenTime);
|
|
|
|
// Act
|
|
var attestation = CreateInTotoAttestation(input, frozenTime, deterministicBundleId);
|
|
|
|
// Assert
|
|
attestation.Should().Contain("\"predicateType\"");
|
|
attestation.Should().Contain("https://stellaops.io/evidence/v1");
|
|
}
|
|
|
|
[Fact]
|
|
public void InTotoAttestation_StatementType_IsConsistent()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleEvidenceInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
var deterministicBundleId = GenerateDeterministicBundleId(input, frozenTime);
|
|
|
|
// Act
|
|
var attestation = CreateInTotoAttestation(input, frozenTime, deterministicBundleId);
|
|
|
|
// Assert
|
|
attestation.Should().Contain("\"_type\"");
|
|
attestation.Should().Contain("https://in-toto.io/Statement/v1");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Evidence Hash Determinism Tests
|
|
|
|
[Fact]
|
|
public void EvidenceHashes_WithIdenticalContent_ProduceDeterministicHashes()
|
|
{
|
|
// Arrange
|
|
var content = "test content for hashing";
|
|
|
|
// Act - Hash the same content multiple times
|
|
var hash1 = ComputeEvidenceHash(content);
|
|
var hash2 = ComputeEvidenceHash(content);
|
|
var hash3 = ComputeEvidenceHash(content);
|
|
|
|
// Assert
|
|
hash1.Should().Be(hash2);
|
|
hash2.Should().Be(hash3);
|
|
hash1.Should().MatchRegex("^sha256:[0-9a-f]{64}$");
|
|
}
|
|
|
|
[Fact]
|
|
public void EvidenceHashSet_Ordering_IsDeterministic()
|
|
{
|
|
// Arrange - Multiple hashes in random order
|
|
var hashes = new[]
|
|
{
|
|
("artifact", "sha256:abcd1234"),
|
|
("sbom", "sha256:efgh5678"),
|
|
("vex", "sha256:ijkl9012"),
|
|
("policy", "sha256:mnop3456")
|
|
};
|
|
|
|
// Act - Create hash sets multiple times
|
|
var hashSet1 = CreateHashSet(hashes);
|
|
var hashSet2 = CreateHashSet(hashes);
|
|
|
|
// Assert - Serialized hash sets should be identical
|
|
var json1 = SerializeHashSet(hashSet1);
|
|
var json2 = SerializeHashSet(hashSet2);
|
|
|
|
json1.Should().Be(json2);
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Completeness Score Determinism Tests
|
|
|
|
[Theory]
|
|
[InlineData(true, true, true, true, 4)]
|
|
[InlineData(true, true, true, false, 3)]
|
|
[InlineData(true, true, false, false, 2)]
|
|
[InlineData(true, false, false, false, 1)]
|
|
[InlineData(false, false, false, false, 0)]
|
|
public void CompletenessScore_IsDeterministic(
|
|
bool hasReachability,
|
|
bool hasCallStack,
|
|
bool hasProvenance,
|
|
bool hasVexStatus,
|
|
int expectedScore)
|
|
{
|
|
// Arrange
|
|
var input = new EvidenceInput
|
|
{
|
|
AlertId = "ALERT-001",
|
|
ArtifactId = "sha256:abc123",
|
|
FindingId = "CVE-2024-1234",
|
|
HasReachability = hasReachability,
|
|
HasCallStack = hasCallStack,
|
|
HasProvenance = hasProvenance,
|
|
HasVexStatus = hasVexStatus,
|
|
Subjects = Array.Empty<string>()
|
|
};
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
var deterministicBundleId = GenerateDeterministicBundleId(input, frozenTime);
|
|
|
|
// Act
|
|
var bundle1 = CreateEvidenceBundle(input, frozenTime, deterministicBundleId);
|
|
var bundle2 = CreateEvidenceBundle(input, frozenTime, deterministicBundleId);
|
|
|
|
// Assert - Both should have same completeness score
|
|
bundle1.Should().Contain($"\"completenessScore\": {expectedScore}");
|
|
bundle2.Should().Contain($"\"completenessScore\": {expectedScore}");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Helper Methods
|
|
|
|
private static EvidenceInput CreateSampleEvidenceInput()
|
|
{
|
|
return new EvidenceInput
|
|
{
|
|
AlertId = "ALERT-2024-001",
|
|
ArtifactId = "sha256:abc123def456",
|
|
FindingId = "CVE-2024-1234",
|
|
HasReachability = true,
|
|
HasCallStack = true,
|
|
HasProvenance = true,
|
|
HasVexStatus = true,
|
|
Subjects = new[] { "pkg:oci/myapp@sha256:abc123" }
|
|
};
|
|
}
|
|
|
|
private static EvidenceInput CreateMultiSubjectEvidenceInput()
|
|
{
|
|
return new EvidenceInput
|
|
{
|
|
AlertId = "ALERT-2024-002",
|
|
ArtifactId = "sha256:multi123",
|
|
FindingId = "CVE-2024-5678",
|
|
HasReachability = true,
|
|
HasCallStack = false,
|
|
HasProvenance = true,
|
|
HasVexStatus = false,
|
|
Subjects = new[]
|
|
{
|
|
"pkg:oci/app-c@sha256:ccc",
|
|
"pkg:oci/app-a@sha256:aaa",
|
|
"pkg:oci/app-b@sha256:bbb"
|
|
}
|
|
};
|
|
}
|
|
|
|
private static string GenerateDeterministicBundleId(EvidenceInput input, DateTimeOffset timestamp)
|
|
{
|
|
var seed = $"{input.AlertId}:{input.ArtifactId}:{input.FindingId}:{timestamp:O}";
|
|
var hash = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(seed));
|
|
return hash[..32]; // Use first 32 chars as bundle ID
|
|
}
|
|
|
|
private static string CreateEvidenceBundle(EvidenceInput input, DateTimeOffset timestamp, string bundleId)
|
|
{
|
|
var completenessScore = CalculateCompletenessScore(input);
|
|
var reachabilityStatus = input.HasReachability ? "available" : "unavailable";
|
|
var callStackStatus = input.HasCallStack ? "available" : "unavailable";
|
|
var provenanceStatus = input.HasProvenance ? "available" : "unavailable";
|
|
var vexStatusValue = input.HasVexStatus ? "available" : "unavailable";
|
|
|
|
var artifactHash = ComputeEvidenceHash(input.ArtifactId);
|
|
|
|
return $$"""
|
|
{
|
|
"bundleId": "{{bundleId}}",
|
|
"schemaVersion": "1.0",
|
|
"alertId": "{{input.AlertId}}",
|
|
"artifactId": "{{input.ArtifactId}}",
|
|
"completenessScore": {{completenessScore}},
|
|
"createdAt": "{{timestamp:O}}",
|
|
"hashes": {
|
|
"artifact": "{{artifactHash}}",
|
|
"bundle": "sha256:{{bundleId}}"
|
|
},
|
|
"reachability": {
|
|
"status": "{{reachabilityStatus}}"
|
|
},
|
|
"callStack": {
|
|
"status": "{{callStackStatus}}"
|
|
},
|
|
"provenance": {
|
|
"status": "{{provenanceStatus}}"
|
|
},
|
|
"vexStatus": {
|
|
"status": "{{vexStatusValue}}"
|
|
}
|
|
}
|
|
""";
|
|
}
|
|
|
|
private static string CreateDsseEnvelope(string payload, DateTimeOffset timestamp)
|
|
{
|
|
var payloadBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(payload));
|
|
var payloadHash = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(payload));
|
|
|
|
// Note: In production, signature would be computed with actual key
|
|
// For determinism testing, we use a deterministic placeholder
|
|
var deterministicSig = $"sig:{payloadHash[..32]}";
|
|
var sigBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(deterministicSig));
|
|
|
|
return $$"""
|
|
{
|
|
"payloadType": "application/vnd.stellaops.evidence+json",
|
|
"payload": "{{payloadBase64}}",
|
|
"signatures": [
|
|
{
|
|
"keyid": "stellaops-signing-key-v1",
|
|
"sig": "{{sigBase64}}"
|
|
}
|
|
]
|
|
}
|
|
""";
|
|
}
|
|
|
|
private static string ExtractDssePayload(string envelope)
|
|
{
|
|
// Extract base64 payload and decode
|
|
var payloadStart = envelope.IndexOf("\"payload\": \"") + 12;
|
|
var payloadEnd = envelope.IndexOf("\"", payloadStart);
|
|
var payloadBase64 = envelope[payloadStart..payloadEnd];
|
|
return Encoding.UTF8.GetString(Convert.FromBase64String(payloadBase64));
|
|
}
|
|
|
|
private static string CreateInTotoAttestation(EvidenceInput input, DateTimeOffset timestamp, string bundleId)
|
|
{
|
|
var subjects = input.Subjects
|
|
.OrderBy(s => s, StringComparer.Ordinal)
|
|
.Select(s => $$"""
|
|
{
|
|
"name": "{{s}}",
|
|
"digest": {
|
|
"sha256": "{{CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(s))}}"
|
|
}
|
|
}
|
|
""");
|
|
|
|
var bundle = CreateEvidenceBundle(input, timestamp, bundleId);
|
|
|
|
return $$"""
|
|
{
|
|
"_type": "https://in-toto.io/Statement/v1",
|
|
"predicateType": "https://stellaops.io/evidence/v1",
|
|
"subject": [
|
|
{{string.Join(",\n ", subjects)}}
|
|
],
|
|
"predicate": {{bundle}}
|
|
}
|
|
""";
|
|
}
|
|
|
|
private static int CalculateCompletenessScore(EvidenceInput input)
|
|
{
|
|
var score = 0;
|
|
if (input.HasReachability) score++;
|
|
if (input.HasCallStack) score++;
|
|
if (input.HasProvenance) score++;
|
|
if (input.HasVexStatus) score++;
|
|
return score;
|
|
}
|
|
|
|
private static string ComputeEvidenceHash(string content)
|
|
{
|
|
var hash = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(content));
|
|
return $"sha256:{hash}";
|
|
}
|
|
|
|
private static Dictionary<string, string> CreateHashSet((string name, string hash)[] hashes)
|
|
{
|
|
return hashes
|
|
.OrderBy(h => h.name, StringComparer.Ordinal)
|
|
.ToDictionary(h => h.name, h => h.hash);
|
|
}
|
|
|
|
private static string SerializeHashSet(Dictionary<string, string> hashSet)
|
|
{
|
|
var entries = hashSet
|
|
.OrderBy(kvp => kvp.Key, StringComparer.Ordinal)
|
|
.Select(kvp => $"\"{kvp.Key}\": \"{kvp.Value}\"");
|
|
return $"{{\n {string.Join(",\n ", entries)}\n}}";
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region DTOs
|
|
|
|
private sealed record EvidenceInput
|
|
{
|
|
public required string AlertId { get; init; }
|
|
public required string ArtifactId { get; init; }
|
|
public required string FindingId { get; init; }
|
|
public required bool HasReachability { get; init; }
|
|
public required bool HasCallStack { get; init; }
|
|
public required bool HasProvenance { get; init; }
|
|
public required bool HasVexStatus { get; init; }
|
|
public required string[] Subjects { get; init; }
|
|
}
|
|
|
|
#endregion
|
|
}
|