Add determinism tests for verdict artifact generation and update SHA256 sums script

- Implemented comprehensive tests for verdict artifact generation to ensure deterministic outputs across various scenarios, including identical inputs, parallel execution, and change ordering.
- Created helper methods for generating sample verdict inputs and computing canonical hashes.
- Added tests to validate the stability of canonical hashes, proof spine ordering, and summary statistics.
- Introduced a new PowerShell script to update SHA256 sums for files, ensuring accurate hash generation and file integrity checks.
This commit is contained in:
StellaOps Bot
2025-12-24 02:17:34 +02:00
parent e59921374e
commit 7503c19b8f
390 changed files with 37389 additions and 5380 deletions

View File

@@ -0,0 +1,170 @@
// -----------------------------------------------------------------------------
// ScannerOpenApiContractTests.cs
// Sprint: SPRINT_5100_0007_0006_webservice_contract
// Task: WEBSVC-5100-007
// Description: OpenAPI schema contract tests for Scanner.WebService
// -----------------------------------------------------------------------------
using FluentAssertions;
using StellaOps.TestKit;
using StellaOps.TestKit.Fixtures;
using Xunit;
namespace StellaOps.Scanner.WebService.Tests.Contract;
/// <summary>
/// Contract tests for Scanner.WebService OpenAPI schema.
/// Validates that the API contract remains stable and detects breaking changes.
/// </summary>
[Trait("Category", TestCategories.Contract)]
[Collection("ScannerWebService")]
public sealed class ScannerOpenApiContractTests : IClassFixture<ScannerApplicationFactory>
{
private readonly ScannerApplicationFactory _factory;
private readonly string _snapshotPath;
public ScannerOpenApiContractTests(ScannerApplicationFactory factory)
{
_factory = factory;
_snapshotPath = Path.Combine(AppContext.BaseDirectory, "Contract", "Expected", "scanner-openapi.json");
}
/// <summary>
/// Validates that the OpenAPI schema matches the expected snapshot.
/// </summary>
[Fact]
public async Task OpenApiSchema_MatchesSnapshot()
{
await ContractTestHelper.ValidateOpenApiSchemaAsync(_factory, _snapshotPath);
}
/// <summary>
/// Validates that all core Scanner endpoints exist in the schema.
/// </summary>
[Fact]
public async Task OpenApiSchema_ContainsCoreEndpoints()
{
var coreEndpoints = new[]
{
"/api/v1/scans",
"/api/v1/scans/{scanId}",
"/api/v1/sbom",
"/api/v1/sbom/{sbomId}",
"/api/v1/findings",
"/api/v1/reports",
"/api/v1/health",
"/api/v1/health/ready"
};
await ContractTestHelper.ValidateEndpointsExistAsync(_factory, coreEndpoints);
}
/// <summary>
/// Detects breaking changes in the OpenAPI schema.
/// </summary>
[Fact]
public async Task OpenApiSchema_NoBreakingChanges()
{
var changes = await ContractTestHelper.DetectBreakingChangesAsync(_factory, _snapshotPath);
if (changes.HasBreakingChanges)
{
var message = "Breaking API changes detected:\n" +
string.Join("\n", changes.BreakingChanges.Select(c => $" - {c}"));
Assert.Fail(message);
}
// Log non-breaking changes for awareness
if (changes.NonBreakingChanges.Count > 0)
{
Console.WriteLine("Non-breaking API changes detected:");
foreach (var change in changes.NonBreakingChanges)
{
Console.WriteLine($" + {change}");
}
}
}
/// <summary>
/// Validates that security schemes are defined in the schema.
/// </summary>
[Fact]
public async Task OpenApiSchema_HasSecuritySchemes()
{
using var client = _factory.CreateClient();
var response = await client.GetAsync("/swagger/v1/swagger.json");
response.EnsureSuccessStatusCode();
var schemaJson = await response.Content.ReadAsStringAsync();
var schema = System.Text.Json.JsonDocument.Parse(schemaJson);
// Check for security schemes (Bearer token expected)
if (schema.RootElement.TryGetProperty("components", out var components) &&
components.TryGetProperty("securitySchemes", out var securitySchemes))
{
securitySchemes.EnumerateObject().Should().NotBeEmpty(
"OpenAPI schema should define security schemes");
}
}
/// <summary>
/// Validates that error responses are documented in the schema.
/// </summary>
[Fact]
public async Task OpenApiSchema_DocumentsErrorResponses()
{
using var client = _factory.CreateClient();
var response = await client.GetAsync("/swagger/v1/swagger.json");
response.EnsureSuccessStatusCode();
var schemaJson = await response.Content.ReadAsStringAsync();
var schema = System.Text.Json.JsonDocument.Parse(schemaJson);
if (schema.RootElement.TryGetProperty("paths", out var paths))
{
var hasErrorResponses = false;
foreach (var path in paths.EnumerateObject())
{
foreach (var method in path.Value.EnumerateObject())
{
if (method.Value.TryGetProperty("responses", out var responses))
{
// Check for 4xx or 5xx responses
foreach (var resp in responses.EnumerateObject())
{
if (resp.Name.StartsWith("4") || resp.Name.StartsWith("5"))
{
hasErrorResponses = true;
break;
}
}
}
}
if (hasErrorResponses) break;
}
hasErrorResponses.Should().BeTrue(
"OpenAPI schema should document error responses (4xx/5xx)");
}
}
/// <summary>
/// Validates schema determinism: multiple fetches produce identical output.
/// </summary>
[Fact]
public async Task OpenApiSchema_IsDeterministic()
{
var schemas = new List<string>();
for (int i = 0; i < 3; i++)
{
using var client = _factory.CreateClient();
var response = await client.GetAsync("/swagger/v1/swagger.json");
response.EnsureSuccessStatusCode();
schemas.Add(await response.Content.ReadAsStringAsync());
}
schemas.Distinct().Should().HaveCount(1,
"OpenAPI schema should be deterministic across fetches");
}
}