491e883653
- Created `StellaOps.TestKit.Tests` project for unit tests related to determinism. - Implemented `DeterminismManifestTests` to validate deterministic output for canonical bytes and strings, file read/write operations, and error handling for invalid schema versions. - Added `SbomDeterminismTests` to ensure identical inputs produce consistent SBOMs across SPDX 3.0.1 and CycloneDX 1.6/1.7 formats, including parallel execution tests. - Updated project references in `StellaOps.Integration.Determinism` to include the new determinism testing library.
587 lines
18 KiB
C#
587 lines
18 KiB
C#
// -----------------------------------------------------------------------------
|
|
// AirGapBundleDeterminismTests.cs
|
|
// Sprint: SPRINT_5100_0007_0003 - Epic B (Determinism Gate)
|
|
// Task: T7 - AirGap Bundle Export Determinism
|
|
// Description: Tests to validate AirGap bundle generation determinism
|
|
// -----------------------------------------------------------------------------
|
|
|
|
using System.Text;
|
|
using FluentAssertions;
|
|
using StellaOps.Canonical.Json;
|
|
using StellaOps.Testing.Determinism;
|
|
using Xunit;
|
|
|
|
namespace StellaOps.Integration.Determinism;
|
|
|
|
/// <summary>
|
|
/// Determinism validation tests for AirGap bundle generation.
|
|
/// Ensures identical inputs produce identical bundles across:
|
|
/// - NDJSON bundle file generation
|
|
/// - Bundle manifest creation
|
|
/// - Entry trace generation
|
|
/// - Multiple runs with frozen time
|
|
/// - Parallel execution
|
|
/// </summary>
|
|
public class AirGapBundleDeterminismTests
|
|
{
|
|
#region NDJSON Bundle Determinism Tests
|
|
|
|
[Fact]
|
|
public void AirGapBundle_WithIdenticalInput_ProducesDeterministicOutput()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleAirGapInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
|
|
// Act - Generate bundle multiple times
|
|
var bundle1 = GenerateNdjsonBundle(input, frozenTime);
|
|
var bundle2 = GenerateNdjsonBundle(input, frozenTime);
|
|
var bundle3 = GenerateNdjsonBundle(input, frozenTime);
|
|
|
|
// Assert - All outputs should be identical
|
|
bundle1.Should().Be(bundle2);
|
|
bundle2.Should().Be(bundle3);
|
|
}
|
|
|
|
[Fact]
|
|
public void AirGapBundle_CanonicalHash_IsStable()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleAirGapInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
|
|
// Act - Generate bundle and compute canonical hash twice
|
|
var bundle1 = GenerateNdjsonBundle(input, frozenTime);
|
|
var hash1 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(bundle1));
|
|
|
|
var bundle2 = GenerateNdjsonBundle(input, frozenTime);
|
|
var hash2 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(bundle2));
|
|
|
|
// Assert
|
|
hash1.Should().Be(hash2, "Same input should produce same canonical hash");
|
|
hash1.Should().MatchRegex("^[0-9a-f]{64}$");
|
|
}
|
|
|
|
[Fact]
|
|
public void AirGapBundle_DeterminismManifest_CanBeCreated()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleAirGapInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
var bundle = GenerateNdjsonBundle(input, frozenTime);
|
|
var bundleBytes = Encoding.UTF8.GetBytes(bundle);
|
|
|
|
var artifactInfo = new ArtifactInfo
|
|
{
|
|
Type = "airgap-bundle",
|
|
Name = "concelier-airgap-export",
|
|
Version = "1.0.0",
|
|
Format = "NDJSON"
|
|
};
|
|
|
|
var toolchain = new ToolchainInfo
|
|
{
|
|
Platform = ".NET 10.0",
|
|
Components = new[]
|
|
{
|
|
new ComponentInfo { Name = "StellaOps.Concelier", Version = "1.0.0" }
|
|
}
|
|
};
|
|
|
|
// Act - Create determinism manifest
|
|
var manifest = DeterminismManifestWriter.CreateManifest(
|
|
bundleBytes,
|
|
artifactInfo,
|
|
toolchain);
|
|
|
|
// Assert
|
|
manifest.SchemaVersion.Should().Be("1.0");
|
|
manifest.Artifact.Format.Should().Be("NDJSON");
|
|
manifest.CanonicalHash.Algorithm.Should().Be("SHA-256");
|
|
manifest.CanonicalHash.Value.Should().MatchRegex("^[0-9a-f]{64}$");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task AirGapBundle_ParallelGeneration_ProducesDeterministicOutput()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleAirGapInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
|
|
// Act - Generate in parallel 20 times
|
|
var tasks = Enumerable.Range(0, 20)
|
|
.Select(_ => Task.Run(() => GenerateNdjsonBundle(input, frozenTime)))
|
|
.ToArray();
|
|
|
|
var bundles = await Task.WhenAll(tasks);
|
|
|
|
// Assert - All outputs should be identical
|
|
bundles.Should().AllBe(bundles[0]);
|
|
}
|
|
|
|
[Fact]
|
|
public void AirGapBundle_ItemOrdering_IsDeterministic()
|
|
{
|
|
// Arrange - Items in random order
|
|
var input = CreateUnorderedAirGapInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
|
|
// Act - Generate bundle multiple times
|
|
var bundle1 = GenerateNdjsonBundle(input, frozenTime);
|
|
var bundle2 = GenerateNdjsonBundle(input, frozenTime);
|
|
|
|
// Assert - Items should be sorted deterministically
|
|
bundle1.Should().Be(bundle2);
|
|
|
|
// Verify items are lexicographically sorted
|
|
var lines = bundle1.Split('\n', StringSplitOptions.RemoveEmptyEntries);
|
|
var sortedLines = lines.OrderBy(l => l, StringComparer.Ordinal).ToArray();
|
|
lines.Should().BeEquivalentTo(sortedLines, options => options.WithStrictOrdering());
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Bundle Manifest Determinism Tests
|
|
|
|
[Fact]
|
|
public void BundleManifest_WithIdenticalInput_ProducesDeterministicOutput()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleAirGapInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
|
|
// Act - Generate manifest multiple times
|
|
var manifest1 = GenerateBundleManifest(input, frozenTime);
|
|
var manifest2 = GenerateBundleManifest(input, frozenTime);
|
|
var manifest3 = GenerateBundleManifest(input, frozenTime);
|
|
|
|
// Assert - All outputs should be identical
|
|
manifest1.Should().Be(manifest2);
|
|
manifest2.Should().Be(manifest3);
|
|
}
|
|
|
|
[Fact]
|
|
public void BundleManifest_CanonicalHash_IsStable()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleAirGapInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
|
|
// Act
|
|
var manifest1 = GenerateBundleManifest(input, frozenTime);
|
|
var hash1 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(manifest1));
|
|
|
|
var manifest2 = GenerateBundleManifest(input, frozenTime);
|
|
var hash2 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(manifest2));
|
|
|
|
// Assert
|
|
hash1.Should().Be(hash2);
|
|
}
|
|
|
|
[Fact]
|
|
public void BundleManifest_BundleSha256_MatchesNdjsonHash()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleAirGapInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
|
|
// Act
|
|
var bundle = GenerateNdjsonBundle(input, frozenTime);
|
|
var bundleHash = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(bundle));
|
|
var manifest = GenerateBundleManifest(input, frozenTime);
|
|
|
|
// Assert - Manifest should contain matching bundle hash
|
|
manifest.Should().Contain($"\"bundleSha256\": \"{bundleHash}\"");
|
|
}
|
|
|
|
[Fact]
|
|
public void BundleManifest_ItemCount_IsAccurate()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleAirGapInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
|
|
// Act
|
|
var manifest = GenerateBundleManifest(input, frozenTime);
|
|
|
|
// Assert
|
|
manifest.Should().Contain($"\"count\": {input.Items.Length}");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Entry Trace Determinism Tests
|
|
|
|
[Fact]
|
|
public void EntryTrace_WithIdenticalInput_ProducesDeterministicOutput()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleAirGapInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
|
|
// Act - Generate entry trace multiple times
|
|
var trace1 = GenerateEntryTrace(input, frozenTime);
|
|
var trace2 = GenerateEntryTrace(input, frozenTime);
|
|
var trace3 = GenerateEntryTrace(input, frozenTime);
|
|
|
|
// Assert - All outputs should be identical
|
|
trace1.Should().Be(trace2);
|
|
trace2.Should().Be(trace3);
|
|
}
|
|
|
|
[Fact]
|
|
public void EntryTrace_LineNumbers_AreSequential()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleAirGapInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
|
|
// Act
|
|
var trace = GenerateEntryTrace(input, frozenTime);
|
|
|
|
// Assert - Line numbers should be sequential starting from 1
|
|
for (int i = 1; i <= input.Items.Length; i++)
|
|
{
|
|
trace.Should().Contain($"\"lineNumber\": {i}");
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void EntryTrace_ItemHashes_AreCorrect()
|
|
{
|
|
// Arrange
|
|
var input = CreateSampleAirGapInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
|
|
// Act
|
|
var trace = GenerateEntryTrace(input, frozenTime);
|
|
|
|
// Assert - Each item hash should be present
|
|
var sortedItems = input.Items.OrderBy(i => i, StringComparer.Ordinal);
|
|
foreach (var item in sortedItems)
|
|
{
|
|
var expectedHash = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(item));
|
|
trace.Should().Contain(expectedHash);
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Feed Snapshot Determinism Tests
|
|
|
|
[Fact]
|
|
public void FeedSnapshot_WithIdenticalInput_ProducesDeterministicOutput()
|
|
{
|
|
// Arrange
|
|
var input = CreateFeedSnapshotInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
|
|
// Act - Generate snapshot multiple times
|
|
var snapshot1 = GenerateFeedSnapshot(input, frozenTime);
|
|
var snapshot2 = GenerateFeedSnapshot(input, frozenTime);
|
|
var snapshot3 = GenerateFeedSnapshot(input, frozenTime);
|
|
|
|
// Assert - All outputs should be identical
|
|
snapshot1.Should().Be(snapshot2);
|
|
snapshot2.Should().Be(snapshot3);
|
|
}
|
|
|
|
[Fact]
|
|
public void FeedSnapshot_SourceOrdering_IsDeterministic()
|
|
{
|
|
// Arrange - Sources in random order
|
|
var input = CreateFeedSnapshotInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
|
|
// Act
|
|
var snapshot = GenerateFeedSnapshot(input, frozenTime);
|
|
|
|
// Assert - Sources should appear in sorted order
|
|
var sourcePositions = input.Sources
|
|
.OrderBy(s => s, StringComparer.Ordinal)
|
|
.Select(s => snapshot.IndexOf($"\"{s}\""))
|
|
.ToArray();
|
|
|
|
// Positions should be ascending
|
|
for (int i = 1; i < sourcePositions.Length; i++)
|
|
{
|
|
sourcePositions[i].Should().BeGreaterThan(sourcePositions[i - 1]);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void FeedSnapshot_Hash_IsStable()
|
|
{
|
|
// Arrange
|
|
var input = CreateFeedSnapshotInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
|
|
// Act
|
|
var snapshot1 = GenerateFeedSnapshot(input, frozenTime);
|
|
var hash1 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(snapshot1));
|
|
|
|
var snapshot2 = GenerateFeedSnapshot(input, frozenTime);
|
|
var hash2 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(snapshot2));
|
|
|
|
// Assert
|
|
hash1.Should().Be(hash2);
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Policy Pack Bundle Determinism Tests
|
|
|
|
[Fact]
|
|
public void PolicyPackBundle_WithIdenticalInput_ProducesDeterministicOutput()
|
|
{
|
|
// Arrange
|
|
var input = CreatePolicyPackInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
|
|
// Act
|
|
var bundle1 = GeneratePolicyPackBundle(input, frozenTime);
|
|
var bundle2 = GeneratePolicyPackBundle(input, frozenTime);
|
|
|
|
// Assert
|
|
bundle1.Should().Be(bundle2);
|
|
}
|
|
|
|
[Fact]
|
|
public void PolicyPackBundle_RuleOrdering_IsDeterministic()
|
|
{
|
|
// Arrange
|
|
var input = CreatePolicyPackInput();
|
|
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
|
|
|
|
// Act
|
|
var bundle = GeneratePolicyPackBundle(input, frozenTime);
|
|
|
|
// Assert - Rules should appear in sorted order
|
|
var rulePositions = input.Rules
|
|
.OrderBy(r => r.Name, StringComparer.Ordinal)
|
|
.Select(r => bundle.IndexOf($"\"{r.Name}\""))
|
|
.ToArray();
|
|
|
|
for (int i = 1; i < rulePositions.Length; i++)
|
|
{
|
|
rulePositions[i].Should().BeGreaterThan(rulePositions[i - 1]);
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Helper Methods
|
|
|
|
private static AirGapInput CreateSampleAirGapInput()
|
|
{
|
|
return new AirGapInput
|
|
{
|
|
Items = new[]
|
|
{
|
|
"{\"cveId\":\"CVE-2024-0001\",\"source\":\"nvd\"}",
|
|
"{\"cveId\":\"CVE-2024-0002\",\"source\":\"nvd\"}",
|
|
"{\"cveId\":\"CVE-2024-0003\",\"source\":\"osv\"}",
|
|
"{\"cveId\":\"GHSA-0001\",\"source\":\"ghsa\"}"
|
|
}
|
|
};
|
|
}
|
|
|
|
private static AirGapInput CreateUnorderedAirGapInput()
|
|
{
|
|
return new AirGapInput
|
|
{
|
|
Items = new[]
|
|
{
|
|
"{\"cveId\":\"CVE-2024-9999\",\"source\":\"nvd\"}",
|
|
"{\"cveId\":\"CVE-2024-0001\",\"source\":\"nvd\"}",
|
|
"{\"cveId\":\"GHSA-zzzz\",\"source\":\"ghsa\"}",
|
|
"{\"cveId\":\"CVE-2024-5555\",\"source\":\"osv\"}",
|
|
"{\"cveId\":\"GHSA-aaaa\",\"source\":\"ghsa\"}"
|
|
}
|
|
};
|
|
}
|
|
|
|
private static FeedSnapshotInput CreateFeedSnapshotInput()
|
|
{
|
|
return new FeedSnapshotInput
|
|
{
|
|
Sources = new[] { "nvd", "osv", "ghsa", "kev", "epss" },
|
|
SnapshotId = "snapshot-2024-001",
|
|
ItemCounts = new Dictionary<string, int>
|
|
{
|
|
{ "nvd", 25000 },
|
|
{ "osv", 15000 },
|
|
{ "ghsa", 8000 },
|
|
{ "kev", 1200 },
|
|
{ "epss", 250000 }
|
|
}
|
|
};
|
|
}
|
|
|
|
private static PolicyPackInput CreatePolicyPackInput()
|
|
{
|
|
return new PolicyPackInput
|
|
{
|
|
PackId = "policy-pack-2024-001",
|
|
Version = "1.0.0",
|
|
Rules = new[]
|
|
{
|
|
new PolicyRule { Name = "kev-critical-block", Priority = 1, Action = "block" },
|
|
new PolicyRule { Name = "high-cvss-warn", Priority = 2, Action = "warn" },
|
|
new PolicyRule { Name = "default-pass", Priority = 100, Action = "allow" }
|
|
}
|
|
};
|
|
}
|
|
|
|
private static string GenerateNdjsonBundle(AirGapInput input, DateTimeOffset timestamp)
|
|
{
|
|
var sortedItems = input.Items
|
|
.OrderBy(item => item, StringComparer.Ordinal);
|
|
|
|
return string.Join("\n", sortedItems);
|
|
}
|
|
|
|
private static string GenerateBundleManifest(AirGapInput input, DateTimeOffset timestamp)
|
|
{
|
|
var sortedItems = input.Items
|
|
.OrderBy(item => item, StringComparer.Ordinal)
|
|
.ToArray();
|
|
|
|
var bundle = GenerateNdjsonBundle(input, timestamp);
|
|
var bundleHash = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(bundle));
|
|
|
|
var entries = sortedItems.Select((item, index) => new
|
|
{
|
|
lineNumber = index + 1,
|
|
sha256 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(item))
|
|
});
|
|
|
|
var entriesJson = string.Join(",\n ", entries.Select(e =>
|
|
$"{{\"lineNumber\": {e.lineNumber}, \"sha256\": \"{e.sha256}\"}}"));
|
|
|
|
var itemsJson = string.Join(",\n ", sortedItems.Select(i => $"\"{EscapeJson(i)}\""));
|
|
|
|
return $$"""
|
|
{
|
|
"bundleSha256": "{{bundleHash}}",
|
|
"count": {{sortedItems.Length}},
|
|
"createdUtc": "{{timestamp:O}}",
|
|
"entries": [
|
|
{{entriesJson}}
|
|
],
|
|
"items": [
|
|
{{itemsJson}}
|
|
]
|
|
}
|
|
""";
|
|
}
|
|
|
|
private static string GenerateEntryTrace(AirGapInput input, DateTimeOffset timestamp)
|
|
{
|
|
var sortedItems = input.Items
|
|
.OrderBy(item => item, StringComparer.Ordinal)
|
|
.ToArray();
|
|
|
|
var entries = sortedItems.Select((item, index) =>
|
|
$$"""
|
|
{
|
|
"lineNumber": {{index + 1}},
|
|
"sha256": "{{CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(item))}}"
|
|
}
|
|
""");
|
|
|
|
return $$"""
|
|
{
|
|
"createdUtc": "{{timestamp:O}}",
|
|
"entries": [
|
|
{{string.Join(",\n ", entries)}}
|
|
]
|
|
}
|
|
""";
|
|
}
|
|
|
|
private static string GenerateFeedSnapshot(FeedSnapshotInput input, DateTimeOffset timestamp)
|
|
{
|
|
var sortedSources = input.Sources
|
|
.OrderBy(s => s, StringComparer.Ordinal)
|
|
.ToArray();
|
|
|
|
var sourceCounts = sortedSources.Select(s =>
|
|
$"\"{s}\": {input.ItemCounts.GetValueOrDefault(s, 0)}");
|
|
|
|
return $$"""
|
|
{
|
|
"snapshotId": "{{input.SnapshotId}}",
|
|
"createdUtc": "{{timestamp:O}}",
|
|
"sources": [{{string.Join(", ", sortedSources.Select(s => $"\"{s}\""))}}],
|
|
"itemCounts": {
|
|
{{string.Join(",\n ", sourceCounts)}}
|
|
}
|
|
}
|
|
""";
|
|
}
|
|
|
|
private static string GeneratePolicyPackBundle(PolicyPackInput input, DateTimeOffset timestamp)
|
|
{
|
|
var sortedRules = input.Rules
|
|
.OrderBy(r => r.Name, StringComparer.Ordinal)
|
|
.ToArray();
|
|
|
|
var rulesJson = string.Join(",\n ", sortedRules.Select(r =>
|
|
$$"""{"name": "{{r.Name}}", "priority": {{r.Priority}}, "action": "{{r.Action}}"}"""));
|
|
|
|
return $$"""
|
|
{
|
|
"packId": "{{input.PackId}}",
|
|
"version": "{{input.Version}}",
|
|
"createdUtc": "{{timestamp:O}}",
|
|
"rules": [
|
|
{{rulesJson}}
|
|
]
|
|
}
|
|
""";
|
|
}
|
|
|
|
private static string EscapeJson(string value)
|
|
{
|
|
return value
|
|
.Replace("\\", "\\\\")
|
|
.Replace("\"", "\\\"")
|
|
.Replace("\n", "\\n")
|
|
.Replace("\r", "\\r")
|
|
.Replace("\t", "\\t");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region DTOs
|
|
|
|
private sealed record AirGapInput
|
|
{
|
|
public required string[] Items { get; init; }
|
|
}
|
|
|
|
private sealed record FeedSnapshotInput
|
|
{
|
|
public required string[] Sources { get; init; }
|
|
public required string SnapshotId { get; init; }
|
|
public required Dictionary<string, int> ItemCounts { get; init; }
|
|
}
|
|
|
|
private sealed record PolicyPackInput
|
|
{
|
|
public required string PackId { get; init; }
|
|
public required string Version { get; init; }
|
|
public required PolicyRule[] Rules { get; init; }
|
|
}
|
|
|
|
private sealed record PolicyRule
|
|
{
|
|
public required string Name { get; init; }
|
|
public required int Priority { get; init; }
|
|
public required string Action { get; init; }
|
|
}
|
|
|
|
#endregion
|
|
}
|