Add tests for SBOM generation determinism across multiple formats

- Created `StellaOps.TestKit.Tests` project for unit tests related to determinism.
- Implemented `DeterminismManifestTests` to validate deterministic output for canonical bytes and strings, file read/write operations, and error handling for invalid schema versions.
- Added `SbomDeterminismTests` to ensure identical inputs produce consistent SBOMs across SPDX 3.0.1 and CycloneDX 1.6/1.7 formats, including parallel execution tests.
- Updated project references in `StellaOps.Integration.Determinism` to include the new determinism testing library.
This commit is contained in:
master
2025-12-23 18:56:12 +02:00
committed by StellaOps Bot
parent 7ac70ece71
commit 5590a99a1a
381 changed files with 21071 additions and 14678 deletions

View File

@@ -0,0 +1,586 @@
// -----------------------------------------------------------------------------
// AirGapBundleDeterminismTests.cs
// Sprint: SPRINT_5100_0007_0003 - Epic B (Determinism Gate)
// Task: T7 - AirGap Bundle Export Determinism
// Description: Tests to validate AirGap bundle generation determinism
// -----------------------------------------------------------------------------
using System.Text;
using FluentAssertions;
using StellaOps.Canonical.Json;
using StellaOps.Testing.Determinism;
using Xunit;
namespace StellaOps.Integration.Determinism;
/// <summary>
/// Determinism validation tests for AirGap bundle generation.
/// Ensures identical inputs produce identical bundles across:
/// - NDJSON bundle file generation
/// - Bundle manifest creation
/// - Entry trace generation
/// - Multiple runs with frozen time
/// - Parallel execution
/// </summary>
public class AirGapBundleDeterminismTests
{
#region NDJSON Bundle Determinism Tests
[Fact]
public void AirGapBundle_WithIdenticalInput_ProducesDeterministicOutput()
{
// Arrange
var input = CreateSampleAirGapInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Generate bundle multiple times
var bundle1 = GenerateNdjsonBundle(input, frozenTime);
var bundle2 = GenerateNdjsonBundle(input, frozenTime);
var bundle3 = GenerateNdjsonBundle(input, frozenTime);
// Assert - All outputs should be identical
bundle1.Should().Be(bundle2);
bundle2.Should().Be(bundle3);
}
[Fact]
public void AirGapBundle_CanonicalHash_IsStable()
{
// Arrange
var input = CreateSampleAirGapInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Generate bundle and compute canonical hash twice
var bundle1 = GenerateNdjsonBundle(input, frozenTime);
var hash1 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(bundle1));
var bundle2 = GenerateNdjsonBundle(input, frozenTime);
var hash2 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(bundle2));
// Assert
hash1.Should().Be(hash2, "Same input should produce same canonical hash");
hash1.Should().MatchRegex("^[0-9a-f]{64}$");
}
[Fact]
public void AirGapBundle_DeterminismManifest_CanBeCreated()
{
// Arrange
var input = CreateSampleAirGapInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
var bundle = GenerateNdjsonBundle(input, frozenTime);
var bundleBytes = Encoding.UTF8.GetBytes(bundle);
var artifactInfo = new ArtifactInfo
{
Type = "airgap-bundle",
Name = "concelier-airgap-export",
Version = "1.0.0",
Format = "NDJSON"
};
var toolchain = new ToolchainInfo
{
Platform = ".NET 10.0",
Components = new[]
{
new ComponentInfo { Name = "StellaOps.Concelier", Version = "1.0.0" }
}
};
// Act - Create determinism manifest
var manifest = DeterminismManifestWriter.CreateManifest(
bundleBytes,
artifactInfo,
toolchain);
// Assert
manifest.SchemaVersion.Should().Be("1.0");
manifest.Artifact.Format.Should().Be("NDJSON");
manifest.CanonicalHash.Algorithm.Should().Be("SHA-256");
manifest.CanonicalHash.Value.Should().MatchRegex("^[0-9a-f]{64}$");
}
[Fact]
public async Task AirGapBundle_ParallelGeneration_ProducesDeterministicOutput()
{
// Arrange
var input = CreateSampleAirGapInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Generate in parallel 20 times
var tasks = Enumerable.Range(0, 20)
.Select(_ => Task.Run(() => GenerateNdjsonBundle(input, frozenTime)))
.ToArray();
var bundles = await Task.WhenAll(tasks);
// Assert - All outputs should be identical
bundles.Should().AllBe(bundles[0]);
}
[Fact]
public void AirGapBundle_ItemOrdering_IsDeterministic()
{
// Arrange - Items in random order
var input = CreateUnorderedAirGapInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Generate bundle multiple times
var bundle1 = GenerateNdjsonBundle(input, frozenTime);
var bundle2 = GenerateNdjsonBundle(input, frozenTime);
// Assert - Items should be sorted deterministically
bundle1.Should().Be(bundle2);
// Verify items are lexicographically sorted
var lines = bundle1.Split('\n', StringSplitOptions.RemoveEmptyEntries);
var sortedLines = lines.OrderBy(l => l, StringComparer.Ordinal).ToArray();
lines.Should().BeEquivalentTo(sortedLines, options => options.WithStrictOrdering());
}
#endregion
#region Bundle Manifest Determinism Tests
[Fact]
public void BundleManifest_WithIdenticalInput_ProducesDeterministicOutput()
{
// Arrange
var input = CreateSampleAirGapInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Generate manifest multiple times
var manifest1 = GenerateBundleManifest(input, frozenTime);
var manifest2 = GenerateBundleManifest(input, frozenTime);
var manifest3 = GenerateBundleManifest(input, frozenTime);
// Assert - All outputs should be identical
manifest1.Should().Be(manifest2);
manifest2.Should().Be(manifest3);
}
[Fact]
public void BundleManifest_CanonicalHash_IsStable()
{
// Arrange
var input = CreateSampleAirGapInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var manifest1 = GenerateBundleManifest(input, frozenTime);
var hash1 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(manifest1));
var manifest2 = GenerateBundleManifest(input, frozenTime);
var hash2 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(manifest2));
// Assert
hash1.Should().Be(hash2);
}
[Fact]
public void BundleManifest_BundleSha256_MatchesNdjsonHash()
{
// Arrange
var input = CreateSampleAirGapInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var bundle = GenerateNdjsonBundle(input, frozenTime);
var bundleHash = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(bundle));
var manifest = GenerateBundleManifest(input, frozenTime);
// Assert - Manifest should contain matching bundle hash
manifest.Should().Contain($"\"bundleSha256\": \"{bundleHash}\"");
}
[Fact]
public void BundleManifest_ItemCount_IsAccurate()
{
// Arrange
var input = CreateSampleAirGapInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var manifest = GenerateBundleManifest(input, frozenTime);
// Assert
manifest.Should().Contain($"\"count\": {input.Items.Length}");
}
#endregion
#region Entry Trace Determinism Tests
[Fact]
public void EntryTrace_WithIdenticalInput_ProducesDeterministicOutput()
{
// Arrange
var input = CreateSampleAirGapInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Generate entry trace multiple times
var trace1 = GenerateEntryTrace(input, frozenTime);
var trace2 = GenerateEntryTrace(input, frozenTime);
var trace3 = GenerateEntryTrace(input, frozenTime);
// Assert - All outputs should be identical
trace1.Should().Be(trace2);
trace2.Should().Be(trace3);
}
[Fact]
public void EntryTrace_LineNumbers_AreSequential()
{
// Arrange
var input = CreateSampleAirGapInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var trace = GenerateEntryTrace(input, frozenTime);
// Assert - Line numbers should be sequential starting from 1
for (int i = 1; i <= input.Items.Length; i++)
{
trace.Should().Contain($"\"lineNumber\": {i}");
}
}
[Fact]
public void EntryTrace_ItemHashes_AreCorrect()
{
// Arrange
var input = CreateSampleAirGapInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var trace = GenerateEntryTrace(input, frozenTime);
// Assert - Each item hash should be present
var sortedItems = input.Items.OrderBy(i => i, StringComparer.Ordinal);
foreach (var item in sortedItems)
{
var expectedHash = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(item));
trace.Should().Contain(expectedHash);
}
}
#endregion
#region Feed Snapshot Determinism Tests
[Fact]
public void FeedSnapshot_WithIdenticalInput_ProducesDeterministicOutput()
{
// Arrange
var input = CreateFeedSnapshotInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Generate snapshot multiple times
var snapshot1 = GenerateFeedSnapshot(input, frozenTime);
var snapshot2 = GenerateFeedSnapshot(input, frozenTime);
var snapshot3 = GenerateFeedSnapshot(input, frozenTime);
// Assert - All outputs should be identical
snapshot1.Should().Be(snapshot2);
snapshot2.Should().Be(snapshot3);
}
[Fact]
public void FeedSnapshot_SourceOrdering_IsDeterministic()
{
// Arrange - Sources in random order
var input = CreateFeedSnapshotInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var snapshot = GenerateFeedSnapshot(input, frozenTime);
// Assert - Sources should appear in sorted order
var sourcePositions = input.Sources
.OrderBy(s => s, StringComparer.Ordinal)
.Select(s => snapshot.IndexOf($"\"{s}\""))
.ToArray();
// Positions should be ascending
for (int i = 1; i < sourcePositions.Length; i++)
{
sourcePositions[i].Should().BeGreaterThan(sourcePositions[i - 1]);
}
}
[Fact]
public void FeedSnapshot_Hash_IsStable()
{
// Arrange
var input = CreateFeedSnapshotInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var snapshot1 = GenerateFeedSnapshot(input, frozenTime);
var hash1 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(snapshot1));
var snapshot2 = GenerateFeedSnapshot(input, frozenTime);
var hash2 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(snapshot2));
// Assert
hash1.Should().Be(hash2);
}
#endregion
#region Policy Pack Bundle Determinism Tests
[Fact]
public void PolicyPackBundle_WithIdenticalInput_ProducesDeterministicOutput()
{
// Arrange
var input = CreatePolicyPackInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var bundle1 = GeneratePolicyPackBundle(input, frozenTime);
var bundle2 = GeneratePolicyPackBundle(input, frozenTime);
// Assert
bundle1.Should().Be(bundle2);
}
[Fact]
public void PolicyPackBundle_RuleOrdering_IsDeterministic()
{
// Arrange
var input = CreatePolicyPackInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var bundle = GeneratePolicyPackBundle(input, frozenTime);
// Assert - Rules should appear in sorted order
var rulePositions = input.Rules
.OrderBy(r => r.Name, StringComparer.Ordinal)
.Select(r => bundle.IndexOf($"\"{r.Name}\""))
.ToArray();
for (int i = 1; i < rulePositions.Length; i++)
{
rulePositions[i].Should().BeGreaterThan(rulePositions[i - 1]);
}
}
#endregion
#region Helper Methods
private static AirGapInput CreateSampleAirGapInput()
{
return new AirGapInput
{
Items = new[]
{
"{\"cveId\":\"CVE-2024-0001\",\"source\":\"nvd\"}",
"{\"cveId\":\"CVE-2024-0002\",\"source\":\"nvd\"}",
"{\"cveId\":\"CVE-2024-0003\",\"source\":\"osv\"}",
"{\"cveId\":\"GHSA-0001\",\"source\":\"ghsa\"}"
}
};
}
private static AirGapInput CreateUnorderedAirGapInput()
{
return new AirGapInput
{
Items = new[]
{
"{\"cveId\":\"CVE-2024-9999\",\"source\":\"nvd\"}",
"{\"cveId\":\"CVE-2024-0001\",\"source\":\"nvd\"}",
"{\"cveId\":\"GHSA-zzzz\",\"source\":\"ghsa\"}",
"{\"cveId\":\"CVE-2024-5555\",\"source\":\"osv\"}",
"{\"cveId\":\"GHSA-aaaa\",\"source\":\"ghsa\"}"
}
};
}
private static FeedSnapshotInput CreateFeedSnapshotInput()
{
return new FeedSnapshotInput
{
Sources = new[] { "nvd", "osv", "ghsa", "kev", "epss" },
SnapshotId = "snapshot-2024-001",
ItemCounts = new Dictionary<string, int>
{
{ "nvd", 25000 },
{ "osv", 15000 },
{ "ghsa", 8000 },
{ "kev", 1200 },
{ "epss", 250000 }
}
};
}
private static PolicyPackInput CreatePolicyPackInput()
{
return new PolicyPackInput
{
PackId = "policy-pack-2024-001",
Version = "1.0.0",
Rules = new[]
{
new PolicyRule { Name = "kev-critical-block", Priority = 1, Action = "block" },
new PolicyRule { Name = "high-cvss-warn", Priority = 2, Action = "warn" },
new PolicyRule { Name = "default-pass", Priority = 100, Action = "allow" }
}
};
}
private static string GenerateNdjsonBundle(AirGapInput input, DateTimeOffset timestamp)
{
var sortedItems = input.Items
.OrderBy(item => item, StringComparer.Ordinal);
return string.Join("\n", sortedItems);
}
private static string GenerateBundleManifest(AirGapInput input, DateTimeOffset timestamp)
{
var sortedItems = input.Items
.OrderBy(item => item, StringComparer.Ordinal)
.ToArray();
var bundle = GenerateNdjsonBundle(input, timestamp);
var bundleHash = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(bundle));
var entries = sortedItems.Select((item, index) => new
{
lineNumber = index + 1,
sha256 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(item))
});
var entriesJson = string.Join(",\n ", entries.Select(e =>
$"{{\"lineNumber\": {e.lineNumber}, \"sha256\": \"{e.sha256}\"}}"));
var itemsJson = string.Join(",\n ", sortedItems.Select(i => $"\"{EscapeJson(i)}\""));
return $$"""
{
"bundleSha256": "{{bundleHash}}",
"count": {{sortedItems.Length}},
"createdUtc": "{{timestamp:O}}",
"entries": [
{{entriesJson}}
],
"items": [
{{itemsJson}}
]
}
""";
}
private static string GenerateEntryTrace(AirGapInput input, DateTimeOffset timestamp)
{
var sortedItems = input.Items
.OrderBy(item => item, StringComparer.Ordinal)
.ToArray();
var entries = sortedItems.Select((item, index) =>
$$"""
{
"lineNumber": {{index + 1}},
"sha256": "{{CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(item))}}"
}
""");
return $$"""
{
"createdUtc": "{{timestamp:O}}",
"entries": [
{{string.Join(",\n ", entries)}}
]
}
""";
}
private static string GenerateFeedSnapshot(FeedSnapshotInput input, DateTimeOffset timestamp)
{
var sortedSources = input.Sources
.OrderBy(s => s, StringComparer.Ordinal)
.ToArray();
var sourceCounts = sortedSources.Select(s =>
$"\"{s}\": {input.ItemCounts.GetValueOrDefault(s, 0)}");
return $$"""
{
"snapshotId": "{{input.SnapshotId}}",
"createdUtc": "{{timestamp:O}}",
"sources": [{{string.Join(", ", sortedSources.Select(s => $"\"{s}\""))}}],
"itemCounts": {
{{string.Join(",\n ", sourceCounts)}}
}
}
""";
}
private static string GeneratePolicyPackBundle(PolicyPackInput input, DateTimeOffset timestamp)
{
var sortedRules = input.Rules
.OrderBy(r => r.Name, StringComparer.Ordinal)
.ToArray();
var rulesJson = string.Join(",\n ", sortedRules.Select(r =>
$$"""{"name": "{{r.Name}}", "priority": {{r.Priority}}, "action": "{{r.Action}}"}"""));
return $$"""
{
"packId": "{{input.PackId}}",
"version": "{{input.Version}}",
"createdUtc": "{{timestamp:O}}",
"rules": [
{{rulesJson}}
]
}
""";
}
private static string EscapeJson(string value)
{
return value
.Replace("\\", "\\\\")
.Replace("\"", "\\\"")
.Replace("\n", "\\n")
.Replace("\r", "\\r")
.Replace("\t", "\\t");
}
#endregion
#region DTOs
private sealed record AirGapInput
{
public required string[] Items { get; init; }
}
private sealed record FeedSnapshotInput
{
public required string[] Sources { get; init; }
public required string SnapshotId { get; init; }
public required Dictionary<string, int> ItemCounts { get; init; }
}
private sealed record PolicyPackInput
{
public required string PackId { get; init; }
public required string Version { get; init; }
public required PolicyRule[] Rules { get; init; }
}
private sealed record PolicyRule
{
public required string Name { get; init; }
public required int Priority { get; init; }
public required string Action { get; init; }
}
#endregion
}