Add tests for SBOM generation determinism across multiple formats
- Created `StellaOps.TestKit.Tests` project for unit tests related to determinism. - Implemented `DeterminismManifestTests` to validate deterministic output for canonical bytes and strings, file read/write operations, and error handling for invalid schema versions. - Added `SbomDeterminismTests` to ensure identical inputs produce consistent SBOMs across SPDX 3.0.1 and CycloneDX 1.6/1.7 formats, including parallel execution tests. - Updated project references in `StellaOps.Integration.Determinism` to include the new determinism testing library.
This commit is contained in:
master
StellaOps Bot
@@ -0,0 +1,495 @@
|
||||
using FluentAssertions;
|
||||
using StellaOps.Canonical.Json;
|
||||
using StellaOps.TestKit.Determinism;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.TestKit.Tests;
|
||||
|
||||
public sealed class DeterminismManifestTests
|
||||
{
|
||||
[Fact]
|
||||
public void ToCanonicalBytes_WithValidManifest_ProducesDeterministicOutput()
|
||||
{
|
||||
// Arrange
|
||||
var manifest = CreateSampleManifest();
|
||||
|
||||
// Act
|
||||
var bytes1 = DeterminismManifestWriter.ToCanonicalBytes(manifest);
|
||||
var bytes2 = DeterminismManifestWriter.ToCanonicalBytes(manifest);
|
||||
|
||||
// Assert
|
||||
bytes1.Should().Equal(bytes2, "Same manifest should produce identical canonical bytes");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ToCanonicalString_WithValidManifest_ProducesDeterministicString()
|
||||
{
|
||||
// Arrange
|
||||
var manifest = CreateSampleManifest();
|
||||
|
||||
// Act
|
||||
var json1 = DeterminismManifestWriter.ToCanonicalString(manifest);
|
||||
var json2 = DeterminismManifestWriter.ToCanonicalString(manifest);
|
||||
|
||||
// Assert
|
||||
json1.Should().Be(json2, "Same manifest should produce identical canonical JSON string");
|
||||
json1.Should().NotContain("\n", "Canonical JSON should have no newlines");
|
||||
json1.Should().NotContain(" ", "Canonical JSON should have no indentation");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WriteToFile_AndReadFromFile_RoundTripsSuccessfully()
|
||||
{
|
||||
// Arrange
|
||||
var manifest = CreateSampleManifest();
|
||||
var tempFile = Path.GetTempFileName();
|
||||
|
||||
try
|
||||
{
|
||||
// Act - Write
|
||||
DeterminismManifestWriter.WriteToFile(manifest, tempFile);
|
||||
|
||||
// Act - Read
|
||||
var readManifest = DeterminismManifestReader.ReadFromFile(tempFile);
|
||||
|
||||
// Assert
|
||||
readManifest.Should().BeEquivalentTo(manifest);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(tempFile))
|
||||
{
|
||||
File.Delete(tempFile);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WriteToFileAsync_AndReadFromFileAsync_RoundTripsSuccessfully()
|
||||
{
|
||||
// Arrange
|
||||
var manifest = CreateSampleManifest();
|
||||
var tempFile = Path.GetTempFileName();
|
||||
|
||||
try
|
||||
{
|
||||
// Act - Write
|
||||
await DeterminismManifestWriter.WriteToFileAsync(manifest, tempFile);
|
||||
|
||||
// Act - Read
|
||||
var readManifest = await DeterminismManifestReader.ReadFromFileAsync(tempFile);
|
||||
|
||||
// Assert
|
||||
readManifest.Should().BeEquivalentTo(manifest);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(tempFile))
|
||||
{
|
||||
File.Delete(tempFile);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FromBytes_WithValidJson_DeserializesSuccessfully()
|
||||
{
|
||||
// Arrange
|
||||
var manifest = CreateSampleManifest();
|
||||
var bytes = DeterminismManifestWriter.ToCanonicalBytes(manifest);
|
||||
|
||||
// Act
|
||||
var deserialized = DeterminismManifestReader.FromBytes(bytes);
|
||||
|
||||
// Assert
|
||||
deserialized.Should().BeEquivalentTo(manifest);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FromString_WithValidJson_DeserializesSuccessfully()
|
||||
{
|
||||
// Arrange
|
||||
var manifest = CreateSampleManifest();
|
||||
var json = DeterminismManifestWriter.ToCanonicalString(manifest);
|
||||
|
||||
// Act
|
||||
var deserialized = DeterminismManifestReader.FromString(json);
|
||||
|
||||
// Assert
|
||||
deserialized.Should().BeEquivalentTo(manifest);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FromBytes_WithInvalidSchemaVersion_ThrowsInvalidOperationException()
|
||||
{
|
||||
// Arrange
|
||||
var manifest = CreateSampleManifest() with { SchemaVersion = "2.0" };
|
||||
var bytes = DeterminismManifestWriter.ToCanonicalBytes(manifest);
|
||||
|
||||
// Act
|
||||
Action act = () => DeterminismManifestReader.FromBytes(bytes);
|
||||
|
||||
// Assert
|
||||
act.Should().Throw<InvalidOperationException>()
|
||||
.WithMessage("*schema version*2.0*");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TryReadFromFileAsync_WithNonExistentFile_ReturnsNull()
|
||||
{
|
||||
// Arrange
|
||||
var nonExistentPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
|
||||
|
||||
// Act
|
||||
var result = DeterminismManifestReader.TryReadFromFileAsync(nonExistentPath).GetAwaiter().GetResult();
|
||||
|
||||
// Assert
|
||||
result.Should().BeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReadFromFile_WithNonExistentFile_ThrowsFileNotFoundException()
|
||||
{
|
||||
// Arrange
|
||||
var nonExistentPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
|
||||
|
||||
// Act
|
||||
Action act = () => DeterminismManifestReader.ReadFromFile(nonExistentPath);
|
||||
|
||||
// Assert
|
||||
act.Should().Throw<FileNotFoundException>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ComputeCanonicalHash_ProducesDeterministicHash()
|
||||
{
|
||||
// Arrange
|
||||
var manifest = CreateSampleManifest();
|
||||
|
||||
// Act
|
||||
var hash1 = DeterminismManifestWriter.ComputeCanonicalHash(manifest);
|
||||
var hash2 = DeterminismManifestWriter.ComputeCanonicalHash(manifest);
|
||||
|
||||
// Assert
|
||||
hash1.Should().Be(hash2, "Same manifest should produce same hash");
|
||||
hash1.Should().MatchRegex("^[0-9a-f]{64}$", "Hash should be 64-character hex string");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CreateManifest_WithValidInputs_CreatesManifestWithCorrectHash()
|
||||
{
|
||||
// Arrange
|
||||
var artifactBytes = "Test artifact content"u8.ToArray();
|
||||
var artifactInfo = new ArtifactInfo
|
||||
{
|
||||
Type = "sbom",
|
||||
Name = "test-sbom",
|
||||
Version = "1.0.0",
|
||||
Format = "SPDX 3.0.1"
|
||||
};
|
||||
var toolchain = new ToolchainInfo
|
||||
{
|
||||
Platform = ".NET 10.0",
|
||||
Components = new[]
|
||||
{
|
||||
new ComponentInfo { Name = "StellaOps.Scanner", Version = "1.0.0" }
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var manifest = DeterminismManifestWriter.CreateManifest(
|
||||
artifactBytes,
|
||||
artifactInfo,
|
||||
toolchain);
|
||||
|
||||
// Assert
|
||||
manifest.SchemaVersion.Should().Be("1.0");
|
||||
manifest.Artifact.Should().Be(artifactInfo);
|
||||
manifest.Toolchain.Should().Be(toolchain);
|
||||
manifest.CanonicalHash.Algorithm.Should().Be("SHA-256");
|
||||
manifest.CanonicalHash.Encoding.Should().Be("hex");
|
||||
manifest.CanonicalHash.Value.Should().MatchRegex("^[0-9a-f]{64}$");
|
||||
manifest.GeneratedAt.Should().BeCloseTo(DateTimeOffset.UtcNow, TimeSpan.FromSeconds(5));
|
||||
|
||||
// Verify hash is correct
|
||||
var expectedHash = CanonJson.Sha256Hex(artifactBytes);
|
||||
manifest.CanonicalHash.Value.Should().Be(expectedHash);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CreateManifestForJsonArtifact_WithValidInputs_CreatesManifestWithCanonicalHash()
|
||||
{
|
||||
// Arrange
|
||||
var artifact = new { Name = "test", Value = 123, Items = new[] { "a", "b", "c" } };
|
||||
var artifactInfo = new ArtifactInfo
|
||||
{
|
||||
Type = "verdict",
|
||||
Name = "test-verdict",
|
||||
Version = "1.0.0"
|
||||
};
|
||||
var toolchain = new ToolchainInfo
|
||||
{
|
||||
Platform = ".NET 10.0",
|
||||
Components = new[]
|
||||
{
|
||||
new ComponentInfo { Name = "StellaOps.Policy.Engine", Version = "1.0.0" }
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var manifest = DeterminismManifestWriter.CreateManifestForJsonArtifact(
|
||||
artifact,
|
||||
artifactInfo,
|
||||
toolchain);
|
||||
|
||||
// Assert
|
||||
manifest.SchemaVersion.Should().Be("1.0");
|
||||
manifest.Artifact.Should().Be(artifactInfo);
|
||||
manifest.Toolchain.Should().Be(toolchain);
|
||||
manifest.CanonicalHash.Algorithm.Should().Be("SHA-256");
|
||||
manifest.CanonicalHash.Encoding.Should().Be("hex");
|
||||
manifest.CanonicalHash.Value.Should().MatchRegex("^[0-9a-f]{64}$");
|
||||
|
||||
// Verify hash is correct (should use canonical JSON)
|
||||
var expectedHash = CanonJson.Hash(artifact);
|
||||
manifest.CanonicalHash.Value.Should().Be(expectedHash);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CreateManifest_WithInputStamps_IncludesInputStamps()
|
||||
{
|
||||
// Arrange
|
||||
var artifactBytes = "Test artifact"u8.ToArray();
|
||||
var artifactInfo = new ArtifactInfo
|
||||
{
|
||||
Type = "sbom",
|
||||
Name = "test",
|
||||
Version = "1.0.0"
|
||||
};
|
||||
var toolchain = new ToolchainInfo
|
||||
{
|
||||
Platform = ".NET 10.0",
|
||||
Components = new[] { new ComponentInfo { Name = "Scanner", Version = "1.0.0" } }
|
||||
};
|
||||
var inputs = new InputStamps
|
||||
{
|
||||
FeedSnapshotHash = "abc123",
|
||||
PolicyManifestHash = "def456",
|
||||
SourceCodeHash = "789abc"
|
||||
};
|
||||
|
||||
// Act
|
||||
var manifest = DeterminismManifestWriter.CreateManifest(
|
||||
artifactBytes,
|
||||
artifactInfo,
|
||||
toolchain,
|
||||
inputs: inputs);
|
||||
|
||||
// Assert
|
||||
manifest.Inputs.Should().NotBeNull();
|
||||
manifest.Inputs!.FeedSnapshotHash.Should().Be("abc123");
|
||||
manifest.Inputs.PolicyManifestHash.Should().Be("def456");
|
||||
manifest.Inputs.SourceCodeHash.Should().Be("789abc");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CreateManifest_WithReproducibilityMetadata_IncludesMetadata()
|
||||
{
|
||||
// Arrange
|
||||
var artifactBytes = "Test artifact"u8.ToArray();
|
||||
var artifactInfo = new ArtifactInfo
|
||||
{
|
||||
Type = "sbom",
|
||||
Name = "test",
|
||||
Version = "1.0.0"
|
||||
};
|
||||
var toolchain = new ToolchainInfo
|
||||
{
|
||||
Platform = ".NET 10.0",
|
||||
Components = new[] { new ComponentInfo { Name = "Scanner", Version = "1.0.0" } }
|
||||
};
|
||||
var reproducibility = new ReproducibilityMetadata
|
||||
{
|
||||
DeterministicSeed = 42,
|
||||
ClockFixed = true,
|
||||
OrderingGuarantee = "sorted",
|
||||
NormalizationRules = new[] { "UTF-8", "LF line endings", "sorted JSON keys" }
|
||||
};
|
||||
|
||||
// Act
|
||||
var manifest = DeterminismManifestWriter.CreateManifest(
|
||||
artifactBytes,
|
||||
artifactInfo,
|
||||
toolchain,
|
||||
reproducibility: reproducibility);
|
||||
|
||||
// Assert
|
||||
manifest.Reproducibility.Should().NotBeNull();
|
||||
manifest.Reproducibility!.DeterministicSeed.Should().Be(42);
|
||||
manifest.Reproducibility.ClockFixed.Should().BeTrue();
|
||||
manifest.Reproducibility.OrderingGuarantee.Should().Be("sorted");
|
||||
manifest.Reproducibility.NormalizationRules.Should().ContainInOrder("UTF-8", "LF line endings", "sorted JSON keys");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CreateManifest_WithVerificationInfo_IncludesVerification()
|
||||
{
|
||||
// Arrange
|
||||
var artifactBytes = "Test artifact"u8.ToArray();
|
||||
var artifactInfo = new ArtifactInfo
|
||||
{
|
||||
Type = "sbom",
|
||||
Name = "test",
|
||||
Version = "1.0.0"
|
||||
};
|
||||
var toolchain = new ToolchainInfo
|
||||
{
|
||||
Platform = ".NET 10.0",
|
||||
Components = new[] { new ComponentInfo { Name = "Scanner", Version = "1.0.0" } }
|
||||
};
|
||||
var verification = new VerificationInfo
|
||||
{
|
||||
Command = "dotnet run --project Scanner -- scan container alpine:3.18",
|
||||
ExpectedHash = "abc123def456",
|
||||
Baseline = "tests/baselines/sbom-alpine-3.18.determinism.json"
|
||||
};
|
||||
|
||||
// Act
|
||||
var manifest = DeterminismManifestWriter.CreateManifest(
|
||||
artifactBytes,
|
||||
artifactInfo,
|
||||
toolchain,
|
||||
verification: verification);
|
||||
|
||||
// Assert
|
||||
manifest.Verification.Should().NotBeNull();
|
||||
manifest.Verification!.Command.Should().Be("dotnet run --project Scanner -- scan container alpine:3.18");
|
||||
manifest.Verification.ExpectedHash.Should().Be("abc123def456");
|
||||
manifest.Verification.Baseline.Should().Be("tests/baselines/sbom-alpine-3.18.determinism.json");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ManifestSerialization_WithComplexMetadata_PreservesAllFields()
|
||||
{
|
||||
// Arrange
|
||||
var manifest = CreateComplexManifest();
|
||||
|
||||
// Act
|
||||
var json = DeterminismManifestWriter.ToCanonicalString(manifest);
|
||||
var deserialized = DeterminismManifestReader.FromString(json);
|
||||
|
||||
// Assert
|
||||
deserialized.Should().BeEquivalentTo(manifest);
|
||||
}
|
||||
|
||||
private static DeterminismManifest CreateSampleManifest()
|
||||
{
|
||||
return new DeterminismManifest
|
||||
{
|
||||
SchemaVersion = "1.0",
|
||||
Artifact = new ArtifactInfo
|
||||
{
|
||||
Type = "sbom",
|
||||
Name = "test-sbom",
|
||||
Version = "1.0.0",
|
||||
Format = "SPDX 3.0.1"
|
||||
},
|
||||
CanonicalHash = new CanonicalHashInfo
|
||||
{
|
||||
Algorithm = "SHA-256",
|
||||
Value = "abc123def456789012345678901234567890123456789012345678901234",
|
||||
Encoding = "hex"
|
||||
},
|
||||
Toolchain = new ToolchainInfo
|
||||
{
|
||||
Platform = ".NET 10.0",
|
||||
Components = new[]
|
||||
{
|
||||
new ComponentInfo
|
||||
{
|
||||
Name = "StellaOps.Scanner",
|
||||
Version = "1.0.0",
|
||||
Hash = "def456abc789012345678901234567890123456789012345678901234567"
|
||||
}
|
||||
}
|
||||
},
|
||||
GeneratedAt = new DateTimeOffset(2025, 12, 23, 17, 45, 0, TimeSpan.Zero)
|
||||
};
|
||||
}
|
||||
|
||||
private static DeterminismManifest CreateComplexManifest()
|
||||
{
|
||||
return new DeterminismManifest
|
||||
{
|
||||
SchemaVersion = "1.0",
|
||||
Artifact = new ArtifactInfo
|
||||
{
|
||||
Type = "evidence-bundle",
|
||||
Name = "test-bundle",
|
||||
Version = "2.0.0",
|
||||
Format = "DSSE Envelope",
|
||||
Metadata = new Dictionary<string, object?>
|
||||
{
|
||||
["predicateType"] = "https://in-toto.io/attestation/v1",
|
||||
["subject"] = "pkg:docker/alpine@3.18"
|
||||
}
|
||||
},
|
||||
CanonicalHash = new CanonicalHashInfo
|
||||
{
|
||||
Algorithm = "SHA-256",
|
||||
Value = "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef",
|
||||
Encoding = "hex"
|
||||
},
|
||||
Inputs = new InputStamps
|
||||
{
|
||||
FeedSnapshotHash = "feed123abc",
|
||||
PolicyManifestHash = "policy456def",
|
||||
SourceCodeHash = "git789ghi",
|
||||
VexDocumentHashes = new[] { "vex123", "vex456" },
|
||||
Custom = new Dictionary<string, string>
|
||||
{
|
||||
["baselineVersion"] = "1.0.0",
|
||||
["environment"] = "production"
|
||||
}
|
||||
},
|
||||
Toolchain = new ToolchainInfo
|
||||
{
|
||||
Platform = ".NET 10.0",
|
||||
Components = new[]
|
||||
{
|
||||
new ComponentInfo { Name = "StellaOps.Attestor", Version = "2.0.0", Hash = "hash123" },
|
||||
new ComponentInfo { Name = "StellaOps.Signer", Version = "2.1.0" }
|
||||
},
|
||||
Compiler = new CompilerInfo
|
||||
{
|
||||
Name = "Roslyn",
|
||||
Version = "4.8.0"
|
||||
}
|
||||
},
|
||||
GeneratedAt = new DateTimeOffset(2025, 12, 23, 18, 0, 0, TimeSpan.Zero),
|
||||
Reproducibility = new ReproducibilityMetadata
|
||||
{
|
||||
DeterministicSeed = 12345,
|
||||
ClockFixed = true,
|
||||
OrderingGuarantee = "stable",
|
||||
NormalizationRules = new[] { "UTF-8", "LF line endings", "no trailing whitespace" }
|
||||
},
|
||||
Verification = new VerificationInfo
|
||||
{
|
||||
Command = "dotnet test --verify-determinism",
|
||||
ExpectedHash = "abc123def456",
|
||||
Baseline = "baselines/test-bundle.json"
|
||||
},
|
||||
Signatures = new[]
|
||||
{
|
||||
new SignatureInfo
|
||||
{
|
||||
Algorithm = "ES256",
|
||||
KeyId = "signing-key-1",
|
||||
Signature = "base64encodedSig==",
|
||||
Timestamp = new DateTimeOffset(2025, 12, 23, 18, 0, 30, TimeSpan.Zero)
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FluentAssertions" Version="6.12.0" />
|
||||
<PackageReference Include="xunit" Version="2.9.0" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.5.7">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\StellaOps.TestKit\StellaOps.TestKit.csproj" />
|
||||
<ProjectReference Include="..\..\StellaOps.Canonical.Json\StellaOps.Canonical.Json.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
Reference in New Issue
Block a user