Add tests for SBOM generation determinism across multiple formats

- Created `StellaOps.TestKit.Tests` project for unit tests related to determinism.
- Implemented `DeterminismManifestTests` to validate deterministic output for canonical bytes and strings, file read/write operations, and error handling for invalid schema versions.
- Added `SbomDeterminismTests` to ensure identical inputs produce consistent SBOMs across SPDX 3.0.1 and CycloneDX 1.6/1.7 formats, including parallel execution tests.
- Updated project references in `StellaOps.Integration.Determinism` to include the new determinism testing library.
This commit is contained in:
master
2025-12-23 18:56:12 +02:00
committed by StellaOps Bot
parent 7ac70ece71
commit 491e883653
409 changed files with 23797 additions and 17779 deletions

View File

@@ -0,0 +1,429 @@
// =============================================================================
// ScoringApiContractTests.cs
// Sprint: SPRINT_5100_0007_0001_testing_strategy_2026
// Task: TEST-STRAT-5100-005 - Introduce one Pact contract test for critical API
// =============================================================================
using System.Text.Json;
using FluentAssertions;
using PactNet;
using PactNet.Matchers;
using StellaOps.Policy.Engine.Scoring;
using StellaOps.Policy.Scoring;
using Xunit;
using Xunit.Abstractions;
namespace StellaOps.Policy.Engine.Contract.Tests;
/// <summary>
/// Consumer-driven contract tests for the Scoring API.
/// Verifies that the ScoringInput/ScoringEngineResult contract is stable
/// between Policy Engine producers and consumers (Scanner, CLI, etc.).
/// </summary>
/// <remarks>
/// This test generates Pact files that can be used for provider verification.
/// The contract specifies expectations on both request and response shapes.
/// </remarks>
[Trait("Category", "Contract")]
[Trait("Sprint", "5100")]
[Trait("Epic", "TestingStrategy")]
public sealed class ScoringApiContractTests : IAsyncLifetime
{
private readonly ITestOutputHelper _output;
private readonly IPactBuilderV4 _pactBuilder;
private readonly string _pactDir;
public ScoringApiContractTests(ITestOutputHelper output)
{
_output = output;
_pactDir = Path.Combine(
Path.GetTempPath(),
"stellaops-pacts",
DateTime.UtcNow.ToString("yyyyMMdd"));
Directory.CreateDirectory(_pactDir);
var pact = Pact.V4("Scanner", "PolicyEngine", new PactConfig
{
PactDir = _pactDir,
DefaultJsonSettings = new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
WriteIndented = true
}
});
_pactBuilder = pact.WithHttpInteractions();
}
public Task InitializeAsync() => Task.CompletedTask;
public Task DisposeAsync()
{
// Pact files are generated when the builder disposes
return Task.CompletedTask;
}
#region Scoring Input Contract Tests
[Fact(DisplayName = "Consumer expects ScoringInput with required fields")]
public async Task Consumer_Expects_ScoringInput_WithRequiredFields()
{
// Arrange - Define what the consumer (Scanner) expects to send
var expectedInput = new
{
findingId = Match.Type("CVE-2024-12345"),
tenantId = Match.Type("tenant-001"),
profileId = Match.Type("default-profile"),
asOf = Match.Regex(
"2025-12-24T12:00:00+00:00",
@"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[+-]\d{2}:\d{2}$"),
cvssBase = Match.Decimal(7.5m),
cvssVersion = Match.Type("3.1"),
reachability = new
{
hopCount = Match.Integer(2)
},
evidence = new
{
types = Match.MinType(new[] { "Runtime" }, 0)
},
provenance = new
{
level = Match.Type("Unsigned")
},
isKnownExploited = Match.Type(false)
};
// Act - Define the interaction
_pactBuilder
.UponReceiving("a request to score a finding")
.Given("scoring engine is available")
.WithRequest(HttpMethod.Post, "/api/v1/score")
.WithJsonBody(expectedInput)
.WillRespond()
.WithStatus(System.Net.HttpStatusCode.OK)
.WithJsonBody(CreateExpectedResponse());
await _pactBuilder.VerifyAsync(async ctx =>
{
// Simulate consumer making a request
using var httpClient = new HttpClient { BaseAddress = ctx.MockServerUri };
var response = await httpClient.PostAsJsonAsync("/api/v1/score", CreateSampleInput());
response.IsSuccessStatusCode.Should().BeTrue();
});
}
[Fact(DisplayName = "Consumer expects ScoringEngineResult with score fields")]
public async Task Consumer_Expects_ScoringEngineResult_WithScoreFields()
{
// Arrange - Define what the consumer expects to receive
var expectedResponse = new
{
findingId = Match.Type("CVE-2024-12345"),
profileId = Match.Type("default-profile"),
profileVersion = Match.Type("simple-v1.0.0"),
rawScore = Match.Integer(75),
finalScore = Match.Integer(75),
severity = Match.Regex("High", @"^(Critical|High|Medium|Low|Informational)$"),
signalValues = Match.Type(new Dictionary<string, int>
{
{ "baseSeverity", 75 },
{ "reachability", 80 },
{ "evidence", 0 },
{ "provenance", 25 }
}),
signalContributions = Match.Type(new Dictionary<string, double>
{
{ "baseSeverity", 0.25 },
{ "reachability", 0.25 },
{ "evidence", 0.0 },
{ "provenance", 0.25 }
}),
scoringProfile = Match.Regex("Simple", @"^(Simple|Advanced|Custom)$"),
scoredAt = Match.Regex(
"2025-12-24T12:00:00+00:00",
@"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[+-]\d{2}:\d{2}$"),
explain = Match.MinType(new[]
{
new
{
factor = Match.Type("baseSeverity"),
rawValue = Match.Integer(75),
weight = Match.Integer(3000),
contribution = Match.Integer(2250),
note = Match.Type("CVSS 7.5 → basis 75")
}
}, 1)
};
// Act
_pactBuilder
.UponReceiving("a request to score and get detailed result")
.Given("scoring engine is available")
.WithRequest(HttpMethod.Post, "/api/v1/score")
.WithJsonBody(CreateMinimalInputMatcher())
.WillRespond()
.WithStatus(System.Net.HttpStatusCode.OK)
.WithJsonBody(expectedResponse);
await _pactBuilder.VerifyAsync(async ctx =>
{
using var httpClient = new HttpClient { BaseAddress = ctx.MockServerUri };
var response = await httpClient.PostAsJsonAsync("/api/v1/score", CreateSampleInput());
response.IsSuccessStatusCode.Should().BeTrue();
var result = await response.Content.ReadFromJsonAsync<JsonElement>();
result.GetProperty("finalScore").GetInt32().Should().BeGreaterOrEqualTo(0);
result.GetProperty("finalScore").GetInt32().Should().BeLessOrEqualTo(100);
});
}
#endregion
#region Edge Cases Contract Tests
[Fact(DisplayName = "Consumer expects validation error for invalid CVSS")]
public async Task Consumer_Expects_ValidationError_ForInvalidCvss()
{
var invalidInput = new
{
findingId = "CVE-2024-12345",
tenantId = "tenant-001",
profileId = "default-profile",
asOf = "2025-12-24T12:00:00+00:00",
cvssBase = 15.0m, // Invalid: CVSS must be 0-10
cvssVersion = "3.1",
reachability = new { hopCount = 2 },
evidence = new { types = new string[0] },
provenance = new { level = "Unsigned" },
isKnownExploited = false
};
var errorResponse = new
{
error = Match.Type("Validation failed"),
details = Match.Type("CvssBase must be between 0.0 and 10.0")
};
_pactBuilder
.UponReceiving("a request with invalid CVSS score")
.Given("scoring engine is available")
.WithRequest(HttpMethod.Post, "/api/v1/score")
.WithJsonBody(invalidInput)
.WillRespond()
.WithStatus(System.Net.HttpStatusCode.BadRequest)
.WithJsonBody(errorResponse);
await _pactBuilder.VerifyAsync(async ctx =>
{
using var httpClient = new HttpClient { BaseAddress = ctx.MockServerUri };
var response = await httpClient.PostAsJsonAsync("/api/v1/score", invalidInput);
response.StatusCode.Should().Be(System.Net.HttpStatusCode.BadRequest);
});
}
[Fact(DisplayName = "Consumer expects unreachable finding has zero reachability score")]
public async Task Consumer_Expects_UnreachableFinding_HasZeroReachability()
{
var unreachableInput = new
{
findingId = Match.Type("CVE-2024-UNREACHABLE"),
tenantId = Match.Type("tenant-001"),
profileId = Match.Type("default-profile"),
asOf = "2025-12-24T12:00:00+00:00",
cvssBase = Match.Decimal(9.8m),
cvssVersion = Match.Type("3.1"),
reachability = new
{
hopCount = (int?)null // Unreachable
},
evidence = new { types = Match.MinType(new string[0], 0) },
provenance = new { level = Match.Type("Unsigned") },
isKnownExploited = Match.Type(false)
};
var expectedResponse = new
{
findingId = Match.Type("CVE-2024-UNREACHABLE"),
signalValues = new
{
reachability = Match.Integer(0) // Must be 0 for unreachable
}
};
_pactBuilder
.UponReceiving("a request to score an unreachable finding")
.Given("scoring engine is available")
.WithRequest(HttpMethod.Post, "/api/v1/score")
.WithJsonBody(unreachableInput)
.WillRespond()
.WithStatus(System.Net.HttpStatusCode.OK)
.WithJsonBody(expectedResponse);
await _pactBuilder.VerifyAsync(async ctx =>
{
using var httpClient = new HttpClient { BaseAddress = ctx.MockServerUri };
var request = new
{
findingId = "CVE-2024-UNREACHABLE",
tenantId = "tenant-001",
profileId = "default-profile",
asOf = "2025-12-24T12:00:00+00:00",
cvssBase = 9.8m,
cvssVersion = "3.1",
reachability = new { hopCount = (int?)null },
evidence = new { types = new string[0] },
provenance = new { level = "Unsigned" },
isKnownExploited = false
};
var response = await httpClient.PostAsJsonAsync("/api/v1/score", request);
response.IsSuccessStatusCode.Should().BeTrue();
});
}
#endregion
#region Helper Methods
private static object CreateExpectedResponse()
{
return new
{
findingId = Match.Type("CVE-2024-12345"),
profileId = Match.Type("default-profile"),
profileVersion = Match.Type("simple-v1.0.0"),
rawScore = Match.Integer(75),
finalScore = Match.Integer(75),
severity = Match.Type("High"),
signalValues = new Dictionary<string, object>
{
{ "baseSeverity", Match.Integer(75) },
{ "reachability", Match.Integer(80) }
},
signalContributions = new Dictionary<string, object>
{
{ "baseSeverity", Match.Decimal(0.25) },
{ "reachability", Match.Decimal(0.25) }
},
scoringProfile = Match.Type("Simple"),
scoredAt = Match.Type("2025-12-24T12:00:00+00:00"),
explain = Match.MinType(new object[0], 0)
};
}
private static object CreateMinimalInputMatcher()
{
return new
{
findingId = Match.Type("CVE-2024-12345"),
tenantId = Match.Type("tenant-001"),
profileId = Match.Type("default-profile"),
asOf = Match.Type("2025-12-24T12:00:00+00:00"),
cvssBase = Match.Decimal(7.5m),
reachability = new { hopCount = Match.Integer(2) },
evidence = new { types = Match.MinType(new string[0], 0) },
provenance = new { level = Match.Type("Unsigned") },
isKnownExploited = Match.Type(false)
};
}
private static object CreateSampleInput()
{
return new
{
findingId = "CVE-2024-12345",
tenantId = "tenant-001",
profileId = "default-profile",
asOf = "2025-12-24T12:00:00+00:00",
cvssBase = 7.5m,
cvssVersion = "3.1",
reachability = new { hopCount = 2 },
evidence = new { types = new[] { "Runtime" } },
provenance = new { level = "Unsigned" },
isKnownExploited = false
};
}
#endregion
}
/// <summary>
/// Additional contract tests for profile-specific behavior.
/// </summary>
[Trait("Category", "Contract")]
[Trait("Sprint", "5100")]
public sealed class ProfileSpecificContractTests : IAsyncLifetime
{
private readonly IPactBuilderV4 _pactBuilder;
private readonly string _pactDir;
public ProfileSpecificContractTests()
{
_pactDir = Path.Combine(
Path.GetTempPath(),
"stellaops-pacts",
DateTime.UtcNow.ToString("yyyyMMdd"));
Directory.CreateDirectory(_pactDir);
var pact = Pact.V4("Scanner", "PolicyEngine", new PactConfig
{
PactDir = _pactDir,
DefaultJsonSettings = new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
}
});
_pactBuilder = pact.WithHttpInteractions();
}
public Task InitializeAsync() => Task.CompletedTask;
public Task DisposeAsync() => Task.CompletedTask;
[Fact(DisplayName = "Consumer expects Simple profile to return Simple in scoringProfile")]
public async Task Consumer_Expects_SimpleProfile_InResponse()
{
_pactBuilder
.UponReceiving("a request for simple profile scoring")
.Given("simple scoring profile is active")
.WithRequest(HttpMethod.Post, "/api/v1/score")
.WithJsonBody(new { profileId = Match.Type("simple") })
.WillRespond()
.WithStatus(System.Net.HttpStatusCode.OK)
.WithJsonBody(new { scoringProfile = Match.Equality("Simple") });
await _pactBuilder.VerifyAsync(async ctx =>
{
using var httpClient = new HttpClient { BaseAddress = ctx.MockServerUri };
var response = await httpClient.PostAsJsonAsync("/api/v1/score", new { profileId = "simple" });
response.IsSuccessStatusCode.Should().BeTrue();
});
}
[Fact(DisplayName = "Consumer expects Advanced profile to return Advanced in scoringProfile")]
public async Task Consumer_Expects_AdvancedProfile_InResponse()
{
_pactBuilder
.UponReceiving("a request for advanced profile scoring")
.Given("advanced scoring profile is active")
.WithRequest(HttpMethod.Post, "/api/v1/score")
.WithJsonBody(new { profileId = Match.Type("advanced") })
.WillRespond()
.WithStatus(System.Net.HttpStatusCode.OK)
.WithJsonBody(new { scoringProfile = Match.Equality("Advanced") });
await _pactBuilder.VerifyAsync(async ctx =>
{
using var httpClient = new HttpClient { BaseAddress = ctx.MockServerUri };
var response = await httpClient.PostAsJsonAsync("/api/v1/score", new { profileId = "advanced" });
response.IsSuccessStatusCode.Should().BeTrue();
});
}
}

View File

@@ -0,0 +1,33 @@
<?xml version='1.0' encoding='utf-8'?>
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>false</TreatWarningsAsErrors>
<IsPackable>false</IsPackable>
<IsTestProject>true</IsTestProject>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
<PackageReference Include="xunit" Version="2.9.3" />
<PackageReference Include="xunit.runner.visualstudio" Version="3.0.1">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="coverlet.collector" Version="6.0.4">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="FluentAssertions" Version="6.12.0" />
<PackageReference Include="PactNet" Version="5.0.0" />
<PackageReference Include="PactNet.Abstractions" Version="5.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../../StellaOps.Policy.Engine/StellaOps.Policy.Engine.csproj" />
<ProjectReference Include="../../../../__Libraries/StellaOps.TestKit/StellaOps.TestKit.csproj" />
</ItemGroup>
</Project>

View File

@@ -1,8 +1,19 @@
// -----------------------------------------------------------------------------
// PolicyPostgresFixture.cs
// Sprint: SPRINT_5100_0007_0004_storage_harness
// Task: STOR-HARNESS-012
// Description: Policy PostgreSQL test fixture using TestKit
// -----------------------------------------------------------------------------
using System.Reflection;
using StellaOps.Infrastructure.Postgres.Testing;
using StellaOps.Policy.Storage.Postgres;
using Xunit;
// Type aliases to disambiguate TestKit and Infrastructure.Postgres.Testing fixtures
using TestKitPostgresFixture = StellaOps.TestKit.Fixtures.PostgresFixture;
using TestKitPostgresIsolationMode = StellaOps.TestKit.Fixtures.PostgresIsolationMode;
namespace StellaOps.Policy.Storage.Postgres.Tests;
/// <summary>
@@ -26,3 +37,36 @@ public sealed class PolicyPostgresCollection : ICollectionFixture<PolicyPostgres
{
public const string Name = "PolicyPostgres";
}
/// <summary>
/// TestKit-based PostgreSQL fixture for Policy storage tests.
/// Uses TestKit's PostgresFixture for enhanced isolation modes.
/// </summary>
public sealed class PolicyTestKitPostgresFixture : IAsyncLifetime
{
private TestKitPostgresFixture _fixture = null!;
private Assembly MigrationAssembly => typeof(PolicyDataSource).Assembly;
public TestKitPostgresFixture Fixture => _fixture;
public string ConnectionString => _fixture.ConnectionString;
public async Task InitializeAsync()
{
_fixture = new TestKitPostgresFixture(TestKitPostgresIsolationMode.Truncation);
await _fixture.InitializeAsync();
await _fixture.ApplyMigrationsFromAssemblyAsync(MigrationAssembly);
}
public Task DisposeAsync() => _fixture.DisposeAsync();
public Task TruncateAllTablesAsync() => _fixture.TruncateAllTablesAsync();
}
/// <summary>
/// Collection definition for Policy TestKit PostgreSQL tests.
/// </summary>
[CollectionDefinition(PolicyTestKitPostgresCollection.Name)]
public sealed class PolicyTestKitPostgresCollection : ICollectionFixture<PolicyTestKitPostgresFixture>
{
public const string Name = "PolicyTestKitPostgres";
}

View File

@@ -30,7 +30,7 @@
<ProjectReference Include="..\..\__Libraries\StellaOps.Policy.Exceptions\StellaOps.Policy.Exceptions.csproj" />
<ProjectReference Include="..\..\__Libraries\StellaOps.Policy.Unknowns\StellaOps.Policy.Unknowns.csproj" />
<ProjectReference Include="..\..\..\__Libraries\StellaOps.Infrastructure.Postgres.Testing\StellaOps.Infrastructure.Postgres.Testing.csproj" />
<ProjectReference Include="..\..\..\__Libraries\StellaOps.TestKit\StellaOps.TestKit.csproj" />
<ProjectReference Include="..\..\StellaOps.Policy.Scoring\StellaOps.Policy.Scoring.csproj" />
</ItemGroup>
</Project>