Add tests for SBOM generation determinism across multiple formats

- Created `StellaOps.TestKit.Tests` project for unit tests related to determinism.
- Implemented `DeterminismManifestTests` to validate deterministic output for canonical bytes and strings, file read/write operations, and error handling for invalid schema versions.
- Added `SbomDeterminismTests` to ensure identical inputs produce consistent SBOMs across SPDX 3.0.1 and CycloneDX 1.6/1.7 formats, including parallel execution tests.
- Updated project references in `StellaOps.Integration.Determinism` to include the new determinism testing library.
This commit is contained in:
master
2025-12-23 18:56:12 +02:00
committed by StellaOps Bot
parent 7ac70ece71
commit 491e883653
409 changed files with 23797 additions and 17779 deletions

View File

@@ -0,0 +1,625 @@
// -----------------------------------------------------------------------------
// VexDeterminismTests.cs
// Sprint: SPRINT_5100_0007_0003 - Epic B (Determinism Gate)
// Task: T4 - VEX Export Determinism (OpenVEX, CSAF)
// Description: Tests to validate VEX generation determinism across formats
// -----------------------------------------------------------------------------
using System.Text;
using FluentAssertions;
using StellaOps.Canonical.Json;
using StellaOps.Testing.Determinism;
using Xunit;
namespace StellaOps.Integration.Determinism;
/// <summary>
/// Determinism validation tests for VEX export generation.
/// Ensures identical inputs produce identical VEX documents across:
/// - OpenVEX format
/// - CSAF 2.0 VEX format
/// - Multiple runs with frozen time
/// - Parallel execution
/// </summary>
public class VexDeterminismTests
{
#region OpenVEX Determinism Tests
[Fact]
public void OpenVex_WithIdenticalInput_ProducesDeterministicOutput()
{
// Arrange
var input = CreateSampleVexInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Generate VEX multiple times
var vex1 = GenerateOpenVex(input, frozenTime);
var vex2 = GenerateOpenVex(input, frozenTime);
var vex3 = GenerateOpenVex(input, frozenTime);
// Assert - All outputs should be identical
vex1.Should().Be(vex2);
vex2.Should().Be(vex3);
}
[Fact]
public void OpenVex_CanonicalHash_IsStable()
{
// Arrange
var input = CreateSampleVexInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Generate VEX and compute canonical hash twice
var vex1 = GenerateOpenVex(input, frozenTime);
var hash1 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(vex1));
var vex2 = GenerateOpenVex(input, frozenTime);
var hash2 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(vex2));
// Assert
hash1.Should().Be(hash2, "Same input should produce same canonical hash");
hash1.Should().MatchRegex("^[0-9a-f]{64}$");
}
[Fact]
public void OpenVex_DeterminismManifest_CanBeCreated()
{
// Arrange
var input = CreateSampleVexInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
var vex = GenerateOpenVex(input, frozenTime);
var vexBytes = Encoding.UTF8.GetBytes(vex);
var artifactInfo = new ArtifactInfo
{
Type = "vex",
Name = "test-container-vex",
Version = "1.0.0",
Format = "OpenVEX"
};
var toolchain = new ToolchainInfo
{
Platform = ".NET 10.0",
Components = new[]
{
new ComponentInfo { Name = "StellaOps.Excititor", Version = "1.0.0" }
}
};
// Act - Create determinism manifest
var manifest = DeterminismManifestWriter.CreateManifest(
vexBytes,
artifactInfo,
toolchain);
// Assert
manifest.SchemaVersion.Should().Be("1.0");
manifest.Artifact.Format.Should().Be("OpenVEX");
manifest.CanonicalHash.Algorithm.Should().Be("SHA-256");
manifest.CanonicalHash.Value.Should().MatchRegex("^[0-9a-f]{64}$");
}
[Fact]
public async Task OpenVex_ParallelGeneration_ProducesDeterministicOutput()
{
// Arrange
var input = CreateSampleVexInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Generate in parallel 20 times
var tasks = Enumerable.Range(0, 20)
.Select(_ => Task.Run(() => GenerateOpenVex(input, frozenTime)))
.ToArray();
var vexDocuments = await Task.WhenAll(tasks);
// Assert - All outputs should be identical
vexDocuments.Should().AllBe(vexDocuments[0]);
}
[Fact]
public void OpenVex_StatementOrdering_IsDeterministic()
{
// Arrange - Multiple claims for different products in random order
var input = CreateMultiStatementVexInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Generate VEX multiple times
var vex1 = GenerateOpenVex(input, frozenTime);
var vex2 = GenerateOpenVex(input, frozenTime);
// Assert - Statement order should be deterministic
vex1.Should().Be(vex2);
vex1.Should().Contain("\"product_ids\"");
}
[Fact]
public void OpenVex_JustificationText_IsCanonicalized()
{
// Arrange - Claims with varying justification text formatting
var input = CreateSampleVexInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var vex = GenerateOpenVex(input, frozenTime);
// Assert - Justification should be present and normalized
vex.Should().Contain("justification");
vex.Should().Contain("inline_mitigations_already_exist");
}
#endregion
#region CSAF 2.0 VEX Determinism Tests
[Fact]
public void CsafVex_WithIdenticalInput_ProducesDeterministicOutput()
{
// Arrange
var input = CreateSampleVexInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Generate VEX multiple times
var vex1 = GenerateCsafVex(input, frozenTime);
var vex2 = GenerateCsafVex(input, frozenTime);
var vex3 = GenerateCsafVex(input, frozenTime);
// Assert - All outputs should be identical
vex1.Should().Be(vex2);
vex2.Should().Be(vex3);
}
[Fact]
public void CsafVex_CanonicalHash_IsStable()
{
// Arrange
var input = CreateSampleVexInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Generate VEX and compute canonical hash twice
var vex1 = GenerateCsafVex(input, frozenTime);
var hash1 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(vex1));
var vex2 = GenerateCsafVex(input, frozenTime);
var hash2 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(vex2));
// Assert
hash1.Should().Be(hash2, "Same input should produce same canonical hash");
hash1.Should().MatchRegex("^[0-9a-f]{64}$");
}
[Fact]
public void CsafVex_DeterminismManifest_CanBeCreated()
{
// Arrange
var input = CreateSampleVexInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
var vex = GenerateCsafVex(input, frozenTime);
var vexBytes = Encoding.UTF8.GetBytes(vex);
var artifactInfo = new ArtifactInfo
{
Type = "vex",
Name = "test-container-vex",
Version = "1.0.0",
Format = "CSAF 2.0"
};
var toolchain = new ToolchainInfo
{
Platform = ".NET 10.0",
Components = new[]
{
new ComponentInfo { Name = "StellaOps.Excititor", Version = "1.0.0" }
}
};
// Act - Create determinism manifest
var manifest = DeterminismManifestWriter.CreateManifest(
vexBytes,
artifactInfo,
toolchain);
// Assert
manifest.SchemaVersion.Should().Be("1.0");
manifest.Artifact.Format.Should().Be("CSAF 2.0");
manifest.CanonicalHash.Algorithm.Should().Be("SHA-256");
manifest.CanonicalHash.Value.Should().MatchRegex("^[0-9a-f]{64}$");
}
[Fact]
public async Task CsafVex_ParallelGeneration_ProducesDeterministicOutput()
{
// Arrange
var input = CreateSampleVexInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Generate in parallel 20 times
var tasks = Enumerable.Range(0, 20)
.Select(_ => Task.Run(() => GenerateCsafVex(input, frozenTime)))
.ToArray();
var vexDocuments = await Task.WhenAll(tasks);
// Assert - All outputs should be identical
vexDocuments.Should().AllBe(vexDocuments[0]);
}
[Fact]
public void CsafVex_VulnerabilityOrdering_IsDeterministic()
{
// Arrange - Multiple vulnerabilities
var input = CreateMultiStatementVexInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Generate VEX multiple times
var vex1 = GenerateCsafVex(input, frozenTime);
var vex2 = GenerateCsafVex(input, frozenTime);
// Assert - Vulnerability order should be deterministic
vex1.Should().Be(vex2);
vex1.Should().Contain("\"vulnerabilities\"");
}
[Fact]
public void CsafVex_ProductTree_IsDeterministic()
{
// Arrange - Multiple products
var input = CreateMultiStatementVexInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var vex = GenerateCsafVex(input, frozenTime);
// Assert - Product tree should be present and ordered
vex.Should().Contain("\"product_tree\"");
vex.Should().Contain("\"branches\"");
}
#endregion
#region Cross-Format Consistency Tests
[Fact]
public void AllVexFormats_WithSameInput_ProduceDifferentButStableHashes()
{
// Arrange
var input = CreateSampleVexInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Generate all formats
var openVex = GenerateOpenVex(input, frozenTime);
var csafVex = GenerateCsafVex(input, frozenTime);
var openVexHash = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(openVex));
var csafVexHash = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(csafVex));
// Assert - Each format should have different hash but be deterministic
openVexHash.Should().NotBe(csafVexHash);
// All hashes should be valid SHA-256
openVexHash.Should().MatchRegex("^[0-9a-f]{64}$");
csafVexHash.Should().MatchRegex("^[0-9a-f]{64}$");
}
[Fact]
public void AllVexFormats_CanProduceDeterminismManifests()
{
// Arrange
var input = CreateSampleVexInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
var toolchain = new ToolchainInfo
{
Platform = ".NET 10.0",
Components = new[]
{
new ComponentInfo { Name = "StellaOps.Excititor", Version = "1.0.0" }
}
};
// Act - Generate manifests for all formats
var formats = new[] { "OpenVEX", "CSAF 2.0" };
var generators = new Func<VexInput, DateTimeOffset, string>[]
{
GenerateOpenVex,
GenerateCsafVex
};
var manifests = formats.Zip(generators)
.Select(pair =>
{
var vex = pair.Second(input, frozenTime);
var vexBytes = Encoding.UTF8.GetBytes(vex);
var artifactInfo = new ArtifactInfo
{
Type = "vex",
Name = "test-container-vex",
Version = "1.0.0",
Format = pair.First
};
return DeterminismManifestWriter.CreateManifest(vexBytes, artifactInfo, toolchain);
})
.ToArray();
// Assert
manifests.Should().HaveCount(2);
manifests.Should().AllSatisfy(m =>
{
m.SchemaVersion.Should().Be("1.0");
m.CanonicalHash.Algorithm.Should().Be("SHA-256");
m.CanonicalHash.Value.Should().MatchRegex("^[0-9a-f]{64}$");
});
}
#endregion
#region Status Transition Determinism Tests
[Fact]
public void VexStatus_NotAffected_IsDeterministic()
{
// Arrange
var input = CreateVexInputWithStatus(VexStatus.NotAffected, "vulnerable_code_not_present");
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var vex1 = GenerateOpenVex(input, frozenTime);
var vex2 = GenerateOpenVex(input, frozenTime);
// Assert
vex1.Should().Be(vex2);
vex1.Should().Contain("not_affected");
vex1.Should().Contain("vulnerable_code_not_present");
}
[Fact]
public void VexStatus_Affected_IsDeterministic()
{
// Arrange
var input = CreateVexInputWithStatus(VexStatus.Affected, null);
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var vex1 = GenerateOpenVex(input, frozenTime);
var vex2 = GenerateOpenVex(input, frozenTime);
// Assert
vex1.Should().Be(vex2);
vex1.Should().Contain("affected");
}
[Fact]
public void VexStatus_Fixed_IsDeterministic()
{
// Arrange
var input = CreateVexInputWithStatus(VexStatus.Fixed, null);
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var vex1 = GenerateOpenVex(input, frozenTime);
var vex2 = GenerateOpenVex(input, frozenTime);
// Assert
vex1.Should().Be(vex2);
vex1.Should().Contain("fixed");
}
[Fact]
public void VexStatus_UnderInvestigation_IsDeterministic()
{
// Arrange
var input = CreateVexInputWithStatus(VexStatus.UnderInvestigation, null);
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var vex1 = GenerateOpenVex(input, frozenTime);
var vex2 = GenerateOpenVex(input, frozenTime);
// Assert
vex1.Should().Be(vex2);
vex1.Should().Contain("under_investigation");
}
#endregion
#region Helper Methods
private static VexInput CreateSampleVexInput()
{
return new VexInput
{
VulnerabilityId = "CVE-2024-1234",
Product = "pkg:oci/myapp@sha256:abc123",
Status = VexStatus.NotAffected,
Justification = "inline_mitigations_already_exist",
ImpactStatement = "The vulnerable code path is not reachable in this deployment.",
Timestamp = DateTimeOffset.Parse("2025-12-23T18:00:00Z")
};
}
private static VexInput CreateMultiStatementVexInput()
{
return new VexInput
{
VulnerabilityId = "CVE-2024-1234",
Product = "pkg:oci/myapp@sha256:abc123",
Status = VexStatus.NotAffected,
Justification = "vulnerable_code_not_present",
ImpactStatement = null,
AdditionalProducts = new[]
{
"pkg:oci/myapp@sha256:def456",
"pkg:oci/myapp@sha256:ghi789"
},
AdditionalVulnerabilities = new[]
{
"CVE-2024-5678",
"CVE-2024-9012"
},
Timestamp = DateTimeOffset.Parse("2025-12-23T18:00:00Z")
};
}
private static VexInput CreateVexInputWithStatus(VexStatus status, string? justification)
{
return new VexInput
{
VulnerabilityId = "CVE-2024-1234",
Product = "pkg:oci/myapp@sha256:abc123",
Status = status,
Justification = justification,
ImpactStatement = null,
Timestamp = DateTimeOffset.Parse("2025-12-23T18:00:00Z")
};
}
private static string GenerateOpenVex(VexInput input, DateTimeOffset timestamp)
{
// TODO: Integrate with actual OpenVexExporter
// For now, return deterministic stub following OpenVEX spec
var deterministicId = GenerateDeterministicId(input, "openvex");
var productIds = new[] { input.Product }
.Concat(input.AdditionalProducts ?? Array.Empty<string>())
.OrderBy(p => p, StringComparer.Ordinal)
.Select(p => $"\"{p}\"");
var vulnerabilities = new[] { input.VulnerabilityId }
.Concat(input.AdditionalVulnerabilities ?? Array.Empty<string>())
.OrderBy(v => v, StringComparer.Ordinal);
var statements = vulnerabilities.Select(vuln =>
$$"""
{
"vulnerability": {"@id": "{{vuln}}"},
"products": [{{string.Join(", ", productIds)}}],
"status": "{{StatusToString(input.Status)}}",
"justification": "{{input.Justification ?? ""}}",
"impact_statement": "{{input.ImpactStatement ?? ""}}"
}
""");
return $$"""
{
"@context": "https://openvex.dev/ns/v0.2.0",
"@id": "{{deterministicId}}",
"author": "StellaOps Excititor",
"timestamp": "{{timestamp:O}}",
"version": 1,
"statements": [
{{string.Join(",\n ", statements)}}
]
}
""";
}
private static string GenerateCsafVex(VexInput input, DateTimeOffset timestamp)
{
// TODO: Integrate with actual CsafExporter
// For now, return deterministic stub following CSAF 2.0 spec
var deterministicId = GenerateDeterministicId(input, "csaf");
var productIds = new[] { input.Product }
.Concat(input.AdditionalProducts ?? Array.Empty<string>())
.OrderBy(p => p, StringComparer.Ordinal);
var vulnerabilities = new[] { input.VulnerabilityId }
.Concat(input.AdditionalVulnerabilities ?? Array.Empty<string>())
.OrderBy(v => v, StringComparer.Ordinal)
.Select(vuln => $$"""
{
"cve": "{{vuln}}",
"product_status": {
"{{CsafStatusCategory(input.Status)}}": [{{string.Join(", ", productIds.Select(p => $"\"{p}\""))}}]
}
}
""");
var branches = productIds.Select(p => $$"""
{
"category": "product_version",
"name": "{{p}}"
}
""");
return $$"""
{
"document": {
"category": "vex",
"csaf_version": "2.0",
"title": "StellaOps VEX CSAF Export",
"publisher": {
"category": "tool",
"name": "StellaOps Excititor"
},
"tracking": {
"id": "{{deterministicId}}",
"status": "final",
"version": "1",
"initial_release_date": "{{timestamp:O}}",
"current_release_date": "{{timestamp:O}}"
}
},
"product_tree": {
"branches": [
{{string.Join(",\n ", branches)}}
]
},
"vulnerabilities": [
{{string.Join(",\n ", vulnerabilities)}}
]
}
""";
}
private static string GenerateDeterministicId(VexInput input, string context)
{
var inputString = $"{context}:{input.VulnerabilityId}:{input.Product}:{input.Status}:{input.Timestamp:O}";
var hash = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(inputString));
return $"urn:uuid:{hash[..8]}-{hash[8..12]}-{hash[12..16]}-{hash[16..20]}-{hash[20..32]}";
}
private static string StatusToString(VexStatus status) => status switch
{
VexStatus.NotAffected => "not_affected",
VexStatus.Affected => "affected",
VexStatus.Fixed => "fixed",
VexStatus.UnderInvestigation => "under_investigation",
_ => "unknown"
};
private static string CsafStatusCategory(VexStatus status) => status switch
{
VexStatus.NotAffected => "known_not_affected",
VexStatus.Affected => "known_affected",
VexStatus.Fixed => "fixed",
VexStatus.UnderInvestigation => "under_investigation",
_ => "unknown"
};
#endregion
#region DTOs
private sealed record VexInput
{
public required string VulnerabilityId { get; init; }
public required string Product { get; init; }
public required VexStatus Status { get; init; }
public string? Justification { get; init; }
public string? ImpactStatement { get; init; }
public string[]? AdditionalProducts { get; init; }
public string[]? AdditionalVulnerabilities { get; init; }
public required DateTimeOffset Timestamp { get; init; }
}
private enum VexStatus
{
NotAffected,
Affected,
Fixed,
UnderInvestigation
}
#endregion
}