Add tests for SBOM generation determinism across multiple formats
- Created `StellaOps.TestKit.Tests` project for unit tests related to determinism. - Implemented `DeterminismManifestTests` to validate deterministic output for canonical bytes and strings, file read/write operations, and error handling for invalid schema versions. - Added `SbomDeterminismTests` to ensure identical inputs produce consistent SBOMs across SPDX 3.0.1 and CycloneDX 1.6/1.7 formats, including parallel execution tests. - Updated project references in `StellaOps.Integration.Determinism` to include the new determinism testing library.
This commit is contained in:
master
StellaOps Bot
@@ -0,0 +1,454 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using StellaOps.Canonical.Json;
|
||||
|
||||
namespace StellaOps.Testing.Determinism;
|
||||
|
||||
/// <summary>
|
||||
/// Stores and retrieves determinism baselines for artifact comparison.
|
||||
/// Baselines are SHA-256 hashes of canonical artifact representations used to detect drift.
|
||||
/// </summary>
|
||||
public sealed class DeterminismBaselineStore
|
||||
{
|
||||
private readonly string _baselineDirectory;
|
||||
private readonly ConcurrentDictionary<string, DeterminismBaseline> _cache = new();
|
||||
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
WriteIndented = true,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) }
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Creates a baseline store with the specified directory.
|
||||
/// </summary>
|
||||
/// <param name="baselineDirectory">Directory path for storing baselines.</param>
|
||||
public DeterminismBaselineStore(string baselineDirectory)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(baselineDirectory);
|
||||
_baselineDirectory = baselineDirectory;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a baseline store using the default baseline directory.
|
||||
/// Default: tests/baselines/determinism relative to repository root.
|
||||
/// </summary>
|
||||
/// <param name="repositoryRoot">Repository root directory.</param>
|
||||
/// <returns>Configured baseline store.</returns>
|
||||
public static DeterminismBaselineStore CreateDefault(string repositoryRoot)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(repositoryRoot);
|
||||
var baselineDir = Path.Combine(repositoryRoot, "tests", "baselines", "determinism");
|
||||
return new DeterminismBaselineStore(baselineDir);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stores a baseline for an artifact.
|
||||
/// </summary>
|
||||
/// <param name="artifactType">Type of artifact (e.g., "sbom", "vex", "policy-verdict").</param>
|
||||
/// <param name="artifactName">Name of the artifact (e.g., "alpine-3.18-spdx").</param>
|
||||
/// <param name="baseline">The baseline to store.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
public async Task StoreBaselineAsync(
|
||||
string artifactType,
|
||||
string artifactName,
|
||||
DeterminismBaseline baseline,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifactType);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifactName);
|
||||
ArgumentNullException.ThrowIfNull(baseline);
|
||||
|
||||
var key = GetBaselineKey(artifactType, artifactName);
|
||||
var filePath = GetBaselineFilePath(artifactType, artifactName);
|
||||
|
||||
// Ensure directory exists
|
||||
var directory = Path.GetDirectoryName(filePath);
|
||||
if (!string.IsNullOrEmpty(directory))
|
||||
{
|
||||
Directory.CreateDirectory(directory);
|
||||
}
|
||||
|
||||
// Serialize and write
|
||||
var json = JsonSerializer.Serialize(baseline, JsonOptions);
|
||||
await File.WriteAllTextAsync(filePath, json, Encoding.UTF8, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Update cache
|
||||
_cache[key] = baseline;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves a baseline for an artifact.
|
||||
/// </summary>
|
||||
/// <param name="artifactType">Type of artifact.</param>
|
||||
/// <param name="artifactName">Name of the artifact.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>The baseline if found, null otherwise.</returns>
|
||||
public async Task<DeterminismBaseline?> GetBaselineAsync(
|
||||
string artifactType,
|
||||
string artifactName,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifactType);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifactName);
|
||||
|
||||
var key = GetBaselineKey(artifactType, artifactName);
|
||||
|
||||
// Check cache first
|
||||
if (_cache.TryGetValue(key, out var cached))
|
||||
{
|
||||
return cached;
|
||||
}
|
||||
|
||||
// Load from file
|
||||
var filePath = GetBaselineFilePath(artifactType, artifactName);
|
||||
if (!File.Exists(filePath))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var json = await File.ReadAllTextAsync(filePath, Encoding.UTF8, cancellationToken).ConfigureAwait(false);
|
||||
var baseline = JsonSerializer.Deserialize<DeterminismBaseline>(json, JsonOptions);
|
||||
|
||||
if (baseline is not null)
|
||||
{
|
||||
_cache[key] = baseline;
|
||||
}
|
||||
|
||||
return baseline;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Compares an artifact against its stored baseline.
|
||||
/// </summary>
|
||||
/// <param name="artifactType">Type of artifact.</param>
|
||||
/// <param name="artifactName">Name of the artifact.</param>
|
||||
/// <param name="currentHash">Current SHA-256 hash of the artifact.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Comparison result indicating match, drift, or missing baseline.</returns>
|
||||
public async Task<BaselineComparisonResult> CompareAsync(
|
||||
string artifactType,
|
||||
string artifactName,
|
||||
string currentHash,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifactType);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifactName);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(currentHash);
|
||||
|
||||
var baseline = await GetBaselineAsync(artifactType, artifactName, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (baseline is null)
|
||||
{
|
||||
return new BaselineComparisonResult
|
||||
{
|
||||
ArtifactType = artifactType,
|
||||
ArtifactName = artifactName,
|
||||
Status = BaselineStatus.Missing,
|
||||
CurrentHash = currentHash,
|
||||
BaselineHash = null,
|
||||
Message = $"No baseline found for {artifactType}/{artifactName}. Run with UPDATE_BASELINES=true to create."
|
||||
};
|
||||
}
|
||||
|
||||
var isMatch = string.Equals(baseline.CanonicalHash, currentHash, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
return new BaselineComparisonResult
|
||||
{
|
||||
ArtifactType = artifactType,
|
||||
ArtifactName = artifactName,
|
||||
Status = isMatch ? BaselineStatus.Match : BaselineStatus.Drift,
|
||||
CurrentHash = currentHash,
|
||||
BaselineHash = baseline.CanonicalHash,
|
||||
BaselineVersion = baseline.Version,
|
||||
Message = isMatch
|
||||
? $"Artifact {artifactType}/{artifactName} matches baseline."
|
||||
: $"DRIFT DETECTED: {artifactType}/{artifactName} hash changed from {baseline.CanonicalHash} to {currentHash}."
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Lists all baselines in the store.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Collection of baseline entries.</returns>
|
||||
public async Task<IReadOnlyList<BaselineEntry>> ListBaselinesAsync(
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var entries = new List<BaselineEntry>();
|
||||
|
||||
if (!Directory.Exists(_baselineDirectory))
|
||||
{
|
||||
return entries;
|
||||
}
|
||||
|
||||
var files = Directory.GetFiles(_baselineDirectory, "*.baseline.json", SearchOption.AllDirectories);
|
||||
|
||||
foreach (var file in files)
|
||||
{
|
||||
try
|
||||
{
|
||||
var json = await File.ReadAllTextAsync(file, Encoding.UTF8, cancellationToken).ConfigureAwait(false);
|
||||
var baseline = JsonSerializer.Deserialize<DeterminismBaseline>(json, JsonOptions);
|
||||
|
||||
if (baseline is not null)
|
||||
{
|
||||
var relativePath = Path.GetRelativePath(_baselineDirectory, file);
|
||||
var parts = relativePath.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
|
||||
|
||||
entries.Add(new BaselineEntry
|
||||
{
|
||||
ArtifactType = parts.Length > 1 ? parts[0] : "unknown",
|
||||
ArtifactName = Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension(file)),
|
||||
CanonicalHash = baseline.CanonicalHash,
|
||||
Version = baseline.Version,
|
||||
UpdatedAt = baseline.UpdatedAt,
|
||||
FilePath = file
|
||||
});
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Skip invalid baseline files
|
||||
}
|
||||
}
|
||||
|
||||
return entries.OrderBy(e => e.ArtifactType).ThenBy(e => e.ArtifactName).ToList();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a baseline from an artifact.
|
||||
/// </summary>
|
||||
/// <param name="artifactBytes">The artifact bytes to hash.</param>
|
||||
/// <param name="version">Version identifier for this baseline.</param>
|
||||
/// <param name="metadata">Optional metadata about the baseline.</param>
|
||||
/// <returns>Created baseline.</returns>
|
||||
public static DeterminismBaseline CreateBaseline(
|
||||
ReadOnlySpan<byte> artifactBytes,
|
||||
string version,
|
||||
IReadOnlyDictionary<string, string>? metadata = null)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(version);
|
||||
|
||||
var hash = CanonJson.Sha256Hex(artifactBytes);
|
||||
|
||||
return new DeterminismBaseline
|
||||
{
|
||||
CanonicalHash = hash,
|
||||
Algorithm = "SHA-256",
|
||||
Version = version,
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
Metadata = metadata
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a baseline from a JSON artifact with canonical serialization.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">The artifact type.</typeparam>
|
||||
/// <param name="artifact">The artifact to serialize and hash.</param>
|
||||
/// <param name="version">Version identifier for this baseline.</param>
|
||||
/// <param name="metadata">Optional metadata about the baseline.</param>
|
||||
/// <returns>Created baseline.</returns>
|
||||
public static DeterminismBaseline CreateBaselineFromJson<T>(
|
||||
T artifact,
|
||||
string version,
|
||||
IReadOnlyDictionary<string, string>? metadata = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(artifact);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(version);
|
||||
|
||||
var canonicalBytes = CanonJson.Canonicalize(artifact);
|
||||
var hash = CanonJson.Sha256Hex(canonicalBytes);
|
||||
|
||||
return new DeterminismBaseline
|
||||
{
|
||||
CanonicalHash = hash,
|
||||
Algorithm = "SHA-256",
|
||||
Version = version,
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
Metadata = metadata
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the baseline directory path.
|
||||
/// </summary>
|
||||
public string BaselineDirectory => _baselineDirectory;
|
||||
|
||||
private string GetBaselineFilePath(string artifactType, string artifactName)
|
||||
{
|
||||
var safeType = SanitizePathComponent(artifactType);
|
||||
var safeName = SanitizePathComponent(artifactName);
|
||||
return Path.Combine(_baselineDirectory, safeType, $"{safeName}.baseline.json");
|
||||
}
|
||||
|
||||
private static string GetBaselineKey(string artifactType, string artifactName)
|
||||
{
|
||||
return $"{artifactType}/{artifactName}".ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static string SanitizePathComponent(string component)
|
||||
{
|
||||
var invalid = Path.GetInvalidFileNameChars();
|
||||
var sanitized = new StringBuilder(component.Length);
|
||||
|
||||
foreach (var c in component)
|
||||
{
|
||||
sanitized.Append(invalid.Contains(c) ? '_' : c);
|
||||
}
|
||||
|
||||
return sanitized.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A stored baseline for determinism comparison.
|
||||
/// </summary>
|
||||
public sealed record DeterminismBaseline
|
||||
{
|
||||
/// <summary>
|
||||
/// SHA-256 hash of the canonical artifact representation (hex-encoded).
|
||||
/// </summary>
|
||||
[JsonPropertyName("canonicalHash")]
|
||||
public required string CanonicalHash { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Hash algorithm used (always "SHA-256").
|
||||
/// </summary>
|
||||
[JsonPropertyName("algorithm")]
|
||||
public required string Algorithm { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Version identifier for this baseline (e.g., "1.0.0", git SHA, or timestamp).
|
||||
/// </summary>
|
||||
[JsonPropertyName("version")]
|
||||
public required string Version { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// UTC timestamp when this baseline was created or updated.
|
||||
/// </summary>
|
||||
[JsonPropertyName("updatedAt")]
|
||||
public required DateTimeOffset UpdatedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional metadata about the baseline.
|
||||
/// </summary>
|
||||
[JsonPropertyName("metadata")]
|
||||
public IReadOnlyDictionary<string, string>? Metadata { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of comparing an artifact against its baseline.
|
||||
/// </summary>
|
||||
public sealed record BaselineComparisonResult
|
||||
{
|
||||
/// <summary>
|
||||
/// Type of artifact compared.
|
||||
/// </summary>
|
||||
[JsonPropertyName("artifactType")]
|
||||
public required string ArtifactType { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Name of artifact compared.
|
||||
/// </summary>
|
||||
[JsonPropertyName("artifactName")]
|
||||
public required string ArtifactName { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Comparison status.
|
||||
/// </summary>
|
||||
[JsonPropertyName("status")]
|
||||
public required BaselineStatus Status { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Current hash of the artifact.
|
||||
/// </summary>
|
||||
[JsonPropertyName("currentHash")]
|
||||
public required string CurrentHash { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Baseline hash (null if missing).
|
||||
/// </summary>
|
||||
[JsonPropertyName("baselineHash")]
|
||||
public string? BaselineHash { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Baseline version (null if missing).
|
||||
/// </summary>
|
||||
[JsonPropertyName("baselineVersion")]
|
||||
public string? BaselineVersion { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Human-readable message describing the result.
|
||||
/// </summary>
|
||||
[JsonPropertyName("message")]
|
||||
public required string Message { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Status of a baseline comparison.
|
||||
/// </summary>
|
||||
public enum BaselineStatus
|
||||
{
|
||||
/// <summary>
|
||||
/// Artifact matches baseline hash.
|
||||
/// </summary>
|
||||
Match,
|
||||
|
||||
/// <summary>
|
||||
/// Artifact hash differs from baseline (drift detected).
|
||||
/// </summary>
|
||||
Drift,
|
||||
|
||||
/// <summary>
|
||||
/// No baseline exists for this artifact.
|
||||
/// </summary>
|
||||
Missing
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry in the baseline registry.
|
||||
/// </summary>
|
||||
public sealed record BaselineEntry
|
||||
{
|
||||
/// <summary>
|
||||
/// Type of artifact.
|
||||
/// </summary>
|
||||
[JsonPropertyName("artifactType")]
|
||||
public required string ArtifactType { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Name of artifact.
|
||||
/// </summary>
|
||||
[JsonPropertyName("artifactName")]
|
||||
public required string ArtifactName { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Canonical hash of the baseline.
|
||||
/// </summary>
|
||||
[JsonPropertyName("canonicalHash")]
|
||||
public required string CanonicalHash { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Version identifier.
|
||||
/// </summary>
|
||||
[JsonPropertyName("version")]
|
||||
public required string Version { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// When baseline was last updated.
|
||||
/// </summary>
|
||||
[JsonPropertyName("updatedAt")]
|
||||
public required DateTimeOffset UpdatedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// File path of the baseline.
|
||||
/// </summary>
|
||||
[JsonPropertyName("filePath")]
|
||||
public required string FilePath { get; init; }
|
||||
}
|
||||
Reference in New Issue
Block a user