Add determinism tests for verdict artifact generation and update SHA256 sums script
- Implemented comprehensive tests for verdict artifact generation to ensure deterministic outputs across various scenarios, including identical inputs, parallel execution, and change ordering. - Created helper methods for generating sample verdict inputs and computing canonical hashes. - Added tests to validate the stability of canonical hashes, proof spine ordering, and summary statistics. - Introduced a new PowerShell script to update SHA256 sums for files, ensuring accurate hash generation and file integrity checks.
This commit is contained in:
@@ -0,0 +1,170 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// ScannerOpenApiContractTests.cs
|
||||
// Sprint: SPRINT_5100_0007_0006_webservice_contract
|
||||
// Task: WEBSVC-5100-007
|
||||
// Description: OpenAPI schema contract tests for Scanner.WebService
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using FluentAssertions;
|
||||
using StellaOps.TestKit;
|
||||
using StellaOps.TestKit.Fixtures;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.WebService.Tests.Contract;
|
||||
|
||||
/// <summary>
|
||||
/// Contract tests for Scanner.WebService OpenAPI schema.
|
||||
/// Validates that the API contract remains stable and detects breaking changes.
|
||||
/// </summary>
|
||||
[Trait("Category", TestCategories.Contract)]
|
||||
[Collection("ScannerWebService")]
|
||||
public sealed class ScannerOpenApiContractTests : IClassFixture<ScannerApplicationFactory>
|
||||
{
|
||||
private readonly ScannerApplicationFactory _factory;
|
||||
private readonly string _snapshotPath;
|
||||
|
||||
public ScannerOpenApiContractTests(ScannerApplicationFactory factory)
|
||||
{
|
||||
_factory = factory;
|
||||
_snapshotPath = Path.Combine(AppContext.BaseDirectory, "Contract", "Expected", "scanner-openapi.json");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates that the OpenAPI schema matches the expected snapshot.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task OpenApiSchema_MatchesSnapshot()
|
||||
{
|
||||
await ContractTestHelper.ValidateOpenApiSchemaAsync(_factory, _snapshotPath);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates that all core Scanner endpoints exist in the schema.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task OpenApiSchema_ContainsCoreEndpoints()
|
||||
{
|
||||
var coreEndpoints = new[]
|
||||
{
|
||||
"/api/v1/scans",
|
||||
"/api/v1/scans/{scanId}",
|
||||
"/api/v1/sbom",
|
||||
"/api/v1/sbom/{sbomId}",
|
||||
"/api/v1/findings",
|
||||
"/api/v1/reports",
|
||||
"/api/v1/health",
|
||||
"/api/v1/health/ready"
|
||||
};
|
||||
|
||||
await ContractTestHelper.ValidateEndpointsExistAsync(_factory, coreEndpoints);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Detects breaking changes in the OpenAPI schema.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task OpenApiSchema_NoBreakingChanges()
|
||||
{
|
||||
var changes = await ContractTestHelper.DetectBreakingChangesAsync(_factory, _snapshotPath);
|
||||
|
||||
if (changes.HasBreakingChanges)
|
||||
{
|
||||
var message = "Breaking API changes detected:\n" +
|
||||
string.Join("\n", changes.BreakingChanges.Select(c => $" - {c}"));
|
||||
Assert.Fail(message);
|
||||
}
|
||||
|
||||
// Log non-breaking changes for awareness
|
||||
if (changes.NonBreakingChanges.Count > 0)
|
||||
{
|
||||
Console.WriteLine("Non-breaking API changes detected:");
|
||||
foreach (var change in changes.NonBreakingChanges)
|
||||
{
|
||||
Console.WriteLine($" + {change}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates that security schemes are defined in the schema.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task OpenApiSchema_HasSecuritySchemes()
|
||||
{
|
||||
using var client = _factory.CreateClient();
|
||||
var response = await client.GetAsync("/swagger/v1/swagger.json");
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var schemaJson = await response.Content.ReadAsStringAsync();
|
||||
var schema = System.Text.Json.JsonDocument.Parse(schemaJson);
|
||||
|
||||
// Check for security schemes (Bearer token expected)
|
||||
if (schema.RootElement.TryGetProperty("components", out var components) &&
|
||||
components.TryGetProperty("securitySchemes", out var securitySchemes))
|
||||
{
|
||||
securitySchemes.EnumerateObject().Should().NotBeEmpty(
|
||||
"OpenAPI schema should define security schemes");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates that error responses are documented in the schema.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task OpenApiSchema_DocumentsErrorResponses()
|
||||
{
|
||||
using var client = _factory.CreateClient();
|
||||
var response = await client.GetAsync("/swagger/v1/swagger.json");
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var schemaJson = await response.Content.ReadAsStringAsync();
|
||||
var schema = System.Text.Json.JsonDocument.Parse(schemaJson);
|
||||
|
||||
if (schema.RootElement.TryGetProperty("paths", out var paths))
|
||||
{
|
||||
var hasErrorResponses = false;
|
||||
foreach (var path in paths.EnumerateObject())
|
||||
{
|
||||
foreach (var method in path.Value.EnumerateObject())
|
||||
{
|
||||
if (method.Value.TryGetProperty("responses", out var responses))
|
||||
{
|
||||
// Check for 4xx or 5xx responses
|
||||
foreach (var resp in responses.EnumerateObject())
|
||||
{
|
||||
if (resp.Name.StartsWith("4") || resp.Name.StartsWith("5"))
|
||||
{
|
||||
hasErrorResponses = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (hasErrorResponses) break;
|
||||
}
|
||||
|
||||
hasErrorResponses.Should().BeTrue(
|
||||
"OpenAPI schema should document error responses (4xx/5xx)");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates schema determinism: multiple fetches produce identical output.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task OpenApiSchema_IsDeterministic()
|
||||
{
|
||||
var schemas = new List<string>();
|
||||
|
||||
for (int i = 0; i < 3; i++)
|
||||
{
|
||||
using var client = _factory.CreateClient();
|
||||
var response = await client.GetAsync("/swagger/v1/swagger.json");
|
||||
response.EnsureSuccessStatusCode();
|
||||
schemas.Add(await response.Content.ReadAsStringAsync());
|
||||
}
|
||||
|
||||
schemas.Distinct().Should().HaveCount(1,
|
||||
"OpenAPI schema should be deterministic across fetches");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user