Add tests for SBOM generation determinism across multiple formats
- Created `StellaOps.TestKit.Tests` project for unit tests related to determinism. - Implemented `DeterminismManifestTests` to validate deterministic output for canonical bytes and strings, file read/write operations, and error handling for invalid schema versions. - Added `SbomDeterminismTests` to ensure identical inputs produce consistent SBOMs across SPDX 3.0.1 and CycloneDX 1.6/1.7 formats, including parallel execution tests. - Updated project references in `StellaOps.Integration.Determinism` to include the new determinism testing library.
This commit is contained in:
master
StellaOps Bot
200
src/__Libraries/StellaOps.TestKit/Fixtures/ContractTestHelper.cs
Normal file
200
src/__Libraries/StellaOps.TestKit/Fixtures/ContractTestHelper.cs
Normal file
@@ -0,0 +1,200 @@
|
||||
using System.Text.Json;
|
||||
using FluentAssertions;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.TestKit.Fixtures;
|
||||
|
||||
/// <summary>
|
||||
/// Helpers for API contract testing using OpenAPI schema snapshots.
|
||||
/// </summary>
|
||||
public static class ContractTestHelper
|
||||
{
|
||||
/// <summary>
|
||||
/// Fetches and validates the OpenAPI schema against a snapshot.
|
||||
/// </summary>
|
||||
public static async Task ValidateOpenApiSchemaAsync<TProgram>(
|
||||
WebApplicationFactory<TProgram> factory,
|
||||
string expectedSnapshotPath,
|
||||
string swaggerEndpoint = "/swagger/v1/swagger.json")
|
||||
where TProgram : class
|
||||
{
|
||||
using var client = factory.CreateClient();
|
||||
var response = await client.GetAsync(swaggerEndpoint);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var actualSchema = await response.Content.ReadAsStringAsync();
|
||||
|
||||
if (ShouldUpdateSnapshots())
|
||||
{
|
||||
await UpdateSnapshotAsync(expectedSnapshotPath, actualSchema);
|
||||
return;
|
||||
}
|
||||
|
||||
var expectedSchema = await File.ReadAllTextAsync(expectedSnapshotPath);
|
||||
|
||||
// Normalize both for comparison
|
||||
var actualNormalized = NormalizeOpenApiSchema(actualSchema);
|
||||
var expectedNormalized = NormalizeOpenApiSchema(expectedSchema);
|
||||
|
||||
actualNormalized.Should().Be(expectedNormalized,
|
||||
"OpenAPI schema should match snapshot. Set STELLAOPS_UPDATE_FIXTURES=true to update.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates that the schema contains expected endpoints.
|
||||
/// </summary>
|
||||
public static async Task ValidateEndpointsExistAsync<TProgram>(
|
||||
WebApplicationFactory<TProgram> factory,
|
||||
IEnumerable<string> expectedEndpoints,
|
||||
string swaggerEndpoint = "/swagger/v1/swagger.json")
|
||||
where TProgram : class
|
||||
{
|
||||
using var client = factory.CreateClient();
|
||||
var response = await client.GetAsync(swaggerEndpoint);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var schemaJson = await response.Content.ReadAsStringAsync();
|
||||
var schema = JsonDocument.Parse(schemaJson);
|
||||
var paths = schema.RootElement.GetProperty("paths");
|
||||
|
||||
foreach (var endpoint in expectedEndpoints)
|
||||
{
|
||||
paths.TryGetProperty(endpoint, out _).Should().BeTrue(
|
||||
$"Expected endpoint '{endpoint}' should exist in OpenAPI schema");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Detects breaking changes between current schema and snapshot.
|
||||
/// </summary>
|
||||
public static async Task<SchemaBreakingChanges> DetectBreakingChangesAsync<TProgram>(
|
||||
WebApplicationFactory<TProgram> factory,
|
||||
string snapshotPath,
|
||||
string swaggerEndpoint = "/swagger/v1/swagger.json")
|
||||
where TProgram : class
|
||||
{
|
||||
using var client = factory.CreateClient();
|
||||
var response = await client.GetAsync(swaggerEndpoint);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var actualSchema = await response.Content.ReadAsStringAsync();
|
||||
|
||||
if (!File.Exists(snapshotPath))
|
||||
{
|
||||
return new SchemaBreakingChanges(new List<string> { "No previous snapshot exists" }, new List<string>());
|
||||
}
|
||||
|
||||
var expectedSchema = await File.ReadAllTextAsync(snapshotPath);
|
||||
|
||||
return CompareSchemas(expectedSchema, actualSchema);
|
||||
}
|
||||
|
||||
private static SchemaBreakingChanges CompareSchemas(string expected, string actual)
|
||||
{
|
||||
var breakingChanges = new List<string>();
|
||||
var nonBreakingChanges = new List<string>();
|
||||
|
||||
try
|
||||
{
|
||||
var expectedDoc = JsonDocument.Parse(expected);
|
||||
var actualDoc = JsonDocument.Parse(actual);
|
||||
|
||||
// Check for removed endpoints (breaking)
|
||||
if (expectedDoc.RootElement.TryGetProperty("paths", out var expectedPaths) &&
|
||||
actualDoc.RootElement.TryGetProperty("paths", out var actualPaths))
|
||||
{
|
||||
foreach (var path in expectedPaths.EnumerateObject())
|
||||
{
|
||||
if (!actualPaths.TryGetProperty(path.Name, out _))
|
||||
{
|
||||
breakingChanges.Add($"Endpoint removed: {path.Name}");
|
||||
}
|
||||
else
|
||||
{
|
||||
// Check for removed methods
|
||||
foreach (var method in path.Value.EnumerateObject())
|
||||
{
|
||||
if (!actualPaths.GetProperty(path.Name).TryGetProperty(method.Name, out _))
|
||||
{
|
||||
breakingChanges.Add($"Method removed: {method.Name.ToUpper()} {path.Name}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for new endpoints (non-breaking)
|
||||
foreach (var path in actualPaths.EnumerateObject())
|
||||
{
|
||||
if (!expectedPaths.TryGetProperty(path.Name, out _))
|
||||
{
|
||||
nonBreakingChanges.Add($"Endpoint added: {path.Name}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for removed schemas (breaking)
|
||||
if (expectedDoc.RootElement.TryGetProperty("components", out var expectedComponents) &&
|
||||
expectedComponents.TryGetProperty("schemas", out var expectedSchemas) &&
|
||||
actualDoc.RootElement.TryGetProperty("components", out var actualComponents) &&
|
||||
actualComponents.TryGetProperty("schemas", out var actualSchemas))
|
||||
{
|
||||
foreach (var schema in expectedSchemas.EnumerateObject())
|
||||
{
|
||||
if (!actualSchemas.TryGetProperty(schema.Name, out _))
|
||||
{
|
||||
breakingChanges.Add($"Schema removed: {schema.Name}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
breakingChanges.Add($"Schema parse error: {ex.Message}");
|
||||
}
|
||||
|
||||
return new SchemaBreakingChanges(breakingChanges, nonBreakingChanges);
|
||||
}
|
||||
|
||||
private static string NormalizeOpenApiSchema(string schema)
|
||||
{
|
||||
try
|
||||
{
|
||||
var doc = JsonDocument.Parse(schema);
|
||||
// Remove non-deterministic fields
|
||||
return JsonSerializer.Serialize(doc, new JsonSerializerOptions
|
||||
{
|
||||
WriteIndented = false,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
||||
});
|
||||
}
|
||||
catch
|
||||
{
|
||||
return schema;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool ShouldUpdateSnapshots()
|
||||
{
|
||||
return Environment.GetEnvironmentVariable("STELLAOPS_UPDATE_FIXTURES") == "true";
|
||||
}
|
||||
|
||||
private static async Task UpdateSnapshotAsync(string path, string content)
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
// Pretty-print for readability
|
||||
var doc = JsonDocument.Parse(content);
|
||||
var pretty = JsonSerializer.Serialize(doc, new JsonSerializerOptions { WriteIndented = true });
|
||||
await File.WriteAllTextAsync(path, pretty);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of schema breaking change detection.
|
||||
/// </summary>
|
||||
public sealed record SchemaBreakingChanges(
|
||||
IReadOnlyList<string> BreakingChanges,
|
||||
IReadOnlyList<string> NonBreakingChanges)
|
||||
{
|
||||
public bool HasBreakingChanges => BreakingChanges.Count > 0;
|
||||
}
|
||||
Reference in New Issue
Block a user