Add tests for SBOM generation determinism across multiple formats
- Created `StellaOps.TestKit.Tests` project for unit tests related to determinism. - Implemented `DeterminismManifestTests` to validate deterministic output for canonical bytes and strings, file read/write operations, and error handling for invalid schema versions. - Added `SbomDeterminismTests` to ensure identical inputs produce consistent SBOMs across SPDX 3.0.1 and CycloneDX 1.6/1.7 formats, including parallel execution tests. - Updated project references in `StellaOps.Integration.Determinism` to include the new determinism testing library.
This commit is contained in:
master
@@ -0,0 +1,267 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://stella-ops.org/schemas/determinism-manifest/v1.json",
|
||||
"title": "StellaOps Determinism Manifest",
|
||||
"description": "Manifest tracking artifact reproducibility with canonical bytes hash, version stamps, and toolchain information",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"schemaVersion",
|
||||
"artifact",
|
||||
"canonicalHash",
|
||||
"toolchain",
|
||||
"generatedAt"
|
||||
],
|
||||
"properties": {
|
||||
"schemaVersion": {
|
||||
"type": "string",
|
||||
"const": "1.0",
|
||||
"description": "Version of this manifest schema"
|
||||
},
|
||||
"artifact": {
|
||||
"type": "object",
|
||||
"description": "Artifact being tracked for determinism",
|
||||
"required": ["type", "name", "version"],
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"sbom",
|
||||
"vex",
|
||||
"csaf",
|
||||
"verdict",
|
||||
"evidence-bundle",
|
||||
"airgap-bundle",
|
||||
"advisory-normalized",
|
||||
"attestation",
|
||||
"other"
|
||||
],
|
||||
"description": "Type of artifact"
|
||||
},
|
||||
"name": {
|
||||
"type": "string",
|
||||
"description": "Artifact identifier or name",
|
||||
"minLength": 1
|
||||
},
|
||||
"version": {
|
||||
"type": "string",
|
||||
"description": "Artifact version or timestamp",
|
||||
"minLength": 1
|
||||
},
|
||||
"format": {
|
||||
"type": "string",
|
||||
"description": "Artifact format (e.g., 'SPDX 3.0.1', 'CycloneDX 1.6', 'OpenVEX')",
|
||||
"examples": ["SPDX 3.0.1", "CycloneDX 1.6", "OpenVEX", "CSAF 2.0"]
|
||||
},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
"description": "Additional artifact-specific metadata",
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"canonicalHash": {
|
||||
"type": "object",
|
||||
"description": "Hash of the canonical representation of the artifact",
|
||||
"required": ["algorithm", "value", "encoding"],
|
||||
"properties": {
|
||||
"algorithm": {
|
||||
"type": "string",
|
||||
"enum": ["SHA-256", "SHA-384", "SHA-512"],
|
||||
"description": "Hash algorithm used"
|
||||
},
|
||||
"value": {
|
||||
"type": "string",
|
||||
"description": "Hex-encoded hash value",
|
||||
"pattern": "^[0-9a-f]{64,128}$"
|
||||
},
|
||||
"encoding": {
|
||||
"type": "string",
|
||||
"enum": ["hex", "base64"],
|
||||
"description": "Encoding of the hash value"
|
||||
}
|
||||
}
|
||||
},
|
||||
"inputs": {
|
||||
"type": "object",
|
||||
"description": "Version stamps of all inputs used to generate the artifact",
|
||||
"properties": {
|
||||
"feedSnapshotHash": {
|
||||
"type": "string",
|
||||
"description": "SHA-256 hash of the vulnerability feed snapshot used",
|
||||
"pattern": "^[0-9a-f]{64}$"
|
||||
},
|
||||
"policyManifestHash": {
|
||||
"type": "string",
|
||||
"description": "SHA-256 hash of the policy manifest used",
|
||||
"pattern": "^[0-9a-f]{64}$"
|
||||
},
|
||||
"sourceCodeHash": {
|
||||
"type": "string",
|
||||
"description": "Git commit SHA or source code hash",
|
||||
"pattern": "^[0-9a-f]{40,64}$"
|
||||
},
|
||||
"dependencyLockfileHash": {
|
||||
"type": "string",
|
||||
"description": "Hash of dependency lockfile (e.g., package-lock.json, Cargo.lock)",
|
||||
"pattern": "^[0-9a-f]{64}$"
|
||||
},
|
||||
"baseImageDigest": {
|
||||
"type": "string",
|
||||
"description": "Container base image digest (sha256:...)",
|
||||
"pattern": "^sha256:[0-9a-f]{64}$"
|
||||
},
|
||||
"vexDocumentHashes": {
|
||||
"type": "array",
|
||||
"description": "Hashes of all VEX documents used as input",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"pattern": "^[0-9a-f]{64}$"
|
||||
}
|
||||
},
|
||||
"custom": {
|
||||
"type": "object",
|
||||
"description": "Custom input hashes specific to artifact type",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"toolchain": {
|
||||
"type": "object",
|
||||
"description": "Toolchain version information",
|
||||
"required": ["platform", "components"],
|
||||
"properties": {
|
||||
"platform": {
|
||||
"type": "string",
|
||||
"description": "Runtime platform (e.g., '.NET 10.0', 'Node.js 20.0')",
|
||||
"examples": [".NET 10.0.0", "Node.js 20.11.0", "Python 3.12.1"]
|
||||
},
|
||||
"components": {
|
||||
"type": "array",
|
||||
"description": "Toolchain component versions",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["name", "version"],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string",
|
||||
"description": "Component name",
|
||||
"examples": ["StellaOps.Scanner", "StellaOps.Policy.Engine", "CycloneDX Generator"]
|
||||
},
|
||||
"version": {
|
||||
"type": "string",
|
||||
"description": "Semantic version or git SHA",
|
||||
"examples": ["1.2.3", "2.0.0-beta.1", "abc123def"]
|
||||
},
|
||||
"hash": {
|
||||
"type": "string",
|
||||
"description": "Optional: SHA-256 hash of the component binary",
|
||||
"pattern": "^[0-9a-f]{64}$"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"compiler": {
|
||||
"type": "object",
|
||||
"description": "Compiler information if applicable",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string",
|
||||
"description": "Compiler name (e.g., 'Roslyn', 'rustc')"
|
||||
},
|
||||
"version": {
|
||||
"type": "string",
|
||||
"description": "Compiler version"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"generatedAt": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
"description": "UTC timestamp when artifact was generated (ISO 8601)",
|
||||
"examples": ["2025-12-23T17:45:00Z"]
|
||||
},
|
||||
"reproducibility": {
|
||||
"type": "object",
|
||||
"description": "Reproducibility metadata",
|
||||
"properties": {
|
||||
"deterministicSeed": {
|
||||
"type": "integer",
|
||||
"description": "Deterministic random seed if used",
|
||||
"minimum": 0
|
||||
},
|
||||
"clockFixed": {
|
||||
"type": "boolean",
|
||||
"description": "Whether system clock was fixed during generation"
|
||||
},
|
||||
"orderingGuarantee": {
|
||||
"type": "string",
|
||||
"enum": ["stable", "sorted", "insertion", "unspecified"],
|
||||
"description": "Ordering guarantee for collections in output"
|
||||
},
|
||||
"normalizationRules": {
|
||||
"type": "array",
|
||||
"description": "Normalization rules applied (e.g., 'UTF-8', 'LF line endings', 'no whitespace')",
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"examples": [
|
||||
["UTF-8 encoding", "LF line endings", "sorted JSON keys", "no trailing whitespace"]
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"verification": {
|
||||
"type": "object",
|
||||
"description": "Verification instructions for reproducing the artifact",
|
||||
"properties": {
|
||||
"command": {
|
||||
"type": "string",
|
||||
"description": "Command to regenerate the artifact",
|
||||
"examples": ["dotnet run --project Scanner -- scan container alpine:3.18"]
|
||||
},
|
||||
"expectedHash": {
|
||||
"type": "string",
|
||||
"description": "Expected SHA-256 hash after reproduction",
|
||||
"pattern": "^[0-9a-f]{64}$"
|
||||
},
|
||||
"baseline": {
|
||||
"type": "string",
|
||||
"description": "Baseline manifest file path for regression testing",
|
||||
"examples": ["tests/baselines/sbom-alpine-3.18.determinism.json"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"signatures": {
|
||||
"type": "array",
|
||||
"description": "Optional cryptographic signatures of this manifest",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["algorithm", "keyId", "signature"],
|
||||
"properties": {
|
||||
"algorithm": {
|
||||
"type": "string",
|
||||
"description": "Signature algorithm (e.g., 'ES256', 'RS256')"
|
||||
},
|
||||
"keyId": {
|
||||
"type": "string",
|
||||
"description": "Key identifier used for signing"
|
||||
},
|
||||
"signature": {
|
||||
"type": "string",
|
||||
"description": "Base64-encoded signature"
|
||||
},
|
||||
"timestamp": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
"description": "UTC timestamp when signature was created"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
613
docs/testing/testkit-usage-guide.md
Normal file
613
docs/testing/testkit-usage-guide.md
Normal file
@@ -0,0 +1,613 @@
|
||||
# StellaOps.TestKit Usage Guide
|
||||
|
||||
**Version:** 1.0
|
||||
**Status:** Pilot Release (Wave 4 Complete)
|
||||
**Audience:** StellaOps developers writing unit, integration, and contract tests
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
`StellaOps.TestKit` provides deterministic testing infrastructure for StellaOps modules. It eliminates flaky tests, provides reproducible test primitives, and standardizes fixtures for integration testing.
|
||||
|
||||
### Key Features
|
||||
|
||||
- **Deterministic Time**: Freeze and advance time for reproducible tests
|
||||
- **Deterministic Random**: Seeded random number generation
|
||||
- **Canonical JSON Assertions**: SHA-256 hash verification for determinism
|
||||
- **Snapshot Testing**: Golden master regression testing
|
||||
- **PostgreSQL Fixture**: Testcontainers-based PostgreSQL 16 for integration tests
|
||||
- **Valkey Fixture**: Redis-compatible caching tests
|
||||
- **HTTP Fixture**: In-memory API contract testing
|
||||
- **OpenTelemetry Capture**: Trace and span assertion helpers
|
||||
- **Test Categories**: Standardized trait constants for CI filtering
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
|
||||
Add `StellaOps.TestKit` as a project reference to your test project:
|
||||
|
||||
```xml
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.TestKit/StellaOps.TestKit.csproj" />
|
||||
</ItemGroup>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Quick Start Examples
|
||||
|
||||
### 1. Deterministic Time
|
||||
|
||||
Eliminate flaky tests caused by time-dependent logic:
|
||||
|
||||
```csharp
|
||||
using StellaOps.TestKit.Deterministic;
|
||||
using Xunit;
|
||||
|
||||
[Fact]
|
||||
public void Test_ExpirationLogic()
|
||||
{
|
||||
// Arrange: Fix time at a known UTC timestamp
|
||||
using var time = new DeterministicTime(new DateTime(2026, 1, 15, 10, 30, 0, DateTimeKind.Utc));
|
||||
|
||||
var expiresAt = time.UtcNow.AddHours(24);
|
||||
|
||||
// Act: Advance time to just before expiration
|
||||
time.Advance(TimeSpan.FromHours(23));
|
||||
Assert.False(time.UtcNow > expiresAt);
|
||||
|
||||
// Advance past expiration
|
||||
time.Advance(TimeSpan.FromHours(2));
|
||||
Assert.True(time.UtcNow > expiresAt);
|
||||
}
|
||||
```
|
||||
|
||||
**API Reference:**
|
||||
- `DeterministicTime(DateTime initialUtc)` - Create with fixed start time
|
||||
- `UtcNow` - Get current deterministic time
|
||||
- `Advance(TimeSpan duration)` - Move time forward
|
||||
- `SetTo(DateTime newUtc)` - Jump to specific time
|
||||
|
||||
---
|
||||
|
||||
### 2. Deterministic Random
|
||||
|
||||
Reproducible random sequences for property tests and fuzzing:
|
||||
|
||||
```csharp
|
||||
using StellaOps.TestKit.Deterministic;
|
||||
|
||||
[Fact]
|
||||
public void Test_RandomIdGeneration()
|
||||
{
|
||||
// Arrange: Same seed produces same sequence
|
||||
var random1 = new DeterministicRandom(seed: 42);
|
||||
var random2 = new DeterministicRandom(seed: 42);
|
||||
|
||||
// Act
|
||||
var guid1 = random1.NextGuid();
|
||||
var guid2 = random2.NextGuid();
|
||||
|
||||
// Assert: Reproducible GUIDs
|
||||
Assert.Equal(guid1, guid2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Test_Shuffling()
|
||||
{
|
||||
var random = new DeterministicRandom(seed: 100);
|
||||
var array = new[] { 1, 2, 3, 4, 5 };
|
||||
|
||||
random.Shuffle(array);
|
||||
|
||||
// Deterministic shuffle order
|
||||
Assert.NotEqual(new[] { 1, 2, 3, 4, 5 }, array);
|
||||
}
|
||||
```
|
||||
|
||||
**API Reference:**
|
||||
- `DeterministicRandom(int seed)` - Create with seed
|
||||
- `NextGuid()` - Generate deterministic GUID
|
||||
- `NextString(int length)` - Generate alphanumeric string
|
||||
- `NextInt(int min, int max)` - Generate integer in range
|
||||
- `Shuffle<T>(T[] array)` - Fisher-Yates shuffle
|
||||
|
||||
---
|
||||
|
||||
### 3. Canonical JSON Assertions
|
||||
|
||||
Verify JSON determinism for SBOM, VEX, and attestation outputs:
|
||||
|
||||
```csharp
|
||||
using StellaOps.TestKit.Assertions;
|
||||
|
||||
[Fact]
|
||||
public void Test_SbomDeterminism()
|
||||
{
|
||||
var sbom = new
|
||||
{
|
||||
SpdxVersion = "SPDX-3.0.1",
|
||||
Name = "MySbom",
|
||||
Packages = new[] { new { Name = "Pkg1", Version = "1.0" } }
|
||||
};
|
||||
|
||||
// Verify deterministic serialization
|
||||
CanonicalJsonAssert.IsDeterministic(sbom, iterations: 100);
|
||||
|
||||
// Verify expected hash (golden master)
|
||||
var expectedHash = "abc123..."; // Precomputed SHA-256
|
||||
CanonicalJsonAssert.HasExpectedHash(sbom, expectedHash);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Test_JsonPropertyExists()
|
||||
{
|
||||
var vex = new
|
||||
{
|
||||
Document = new { Id = "VEX-2026-001" },
|
||||
Statements = new[] { new { Vulnerability = "CVE-2026-1234" } }
|
||||
};
|
||||
|
||||
// Deep property verification
|
||||
CanonicalJsonAssert.ContainsProperty(vex, "Document.Id", "VEX-2026-001");
|
||||
CanonicalJsonAssert.ContainsProperty(vex, "Statements[0].Vulnerability", "CVE-2026-1234");
|
||||
}
|
||||
```
|
||||
|
||||
**API Reference:**
|
||||
- `IsDeterministic<T>(T value, int iterations)` - Verify N serializations match
|
||||
- `HasExpectedHash<T>(T value, string expectedSha256Hex)` - Verify SHA-256 hash
|
||||
- `ComputeCanonicalHash<T>(T value)` - Compute hash for golden master
|
||||
- `AreCanonicallyEqual<T>(T expected, T actual)` - Compare canonical JSON
|
||||
- `ContainsProperty<T>(T value, string propertyPath, object expectedValue)` - Deep search
|
||||
|
||||
---
|
||||
|
||||
### 4. Snapshot Testing
|
||||
|
||||
Golden master regression testing for complex outputs:
|
||||
|
||||
```csharp
|
||||
using StellaOps.TestKit.Assertions;
|
||||
|
||||
[Fact, Trait("Category", TestCategories.Snapshot)]
|
||||
public void Test_SbomGeneration()
|
||||
{
|
||||
var sbom = GenerateSbom(); // Your SBOM generation logic
|
||||
|
||||
// Snapshot will be stored in Snapshots/TestSbomGeneration.json
|
||||
SnapshotAssert.MatchesSnapshot(sbom, "TestSbomGeneration");
|
||||
}
|
||||
|
||||
// Update snapshots when intentional changes occur:
|
||||
// UPDATE_SNAPSHOTS=1 dotnet test
|
||||
```
|
||||
|
||||
**Text and Binary Snapshots:**
|
||||
|
||||
```csharp
|
||||
[Fact]
|
||||
public void Test_LicenseText()
|
||||
{
|
||||
var licenseText = GenerateLicenseNotice();
|
||||
SnapshotAssert.MatchesTextSnapshot(licenseText, "LicenseNotice");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Test_SignatureBytes()
|
||||
{
|
||||
var signature = SignDocument(document);
|
||||
SnapshotAssert.MatchesBinarySnapshot(signature, "DocumentSignature");
|
||||
}
|
||||
```
|
||||
|
||||
**API Reference:**
|
||||
- `MatchesSnapshot<T>(T value, string snapshotName)` - JSON snapshot
|
||||
- `MatchesTextSnapshot(string value, string snapshotName)` - Text snapshot
|
||||
- `MatchesBinarySnapshot(byte[] value, string snapshotName)` - Binary snapshot
|
||||
- Environment variable: `UPDATE_SNAPSHOTS=1` to update baselines
|
||||
|
||||
---
|
||||
|
||||
### 5. PostgreSQL Fixture
|
||||
|
||||
Testcontainers-based PostgreSQL 16 for integration tests:
|
||||
|
||||
```csharp
|
||||
using StellaOps.TestKit.Fixtures;
|
||||
using Xunit;
|
||||
|
||||
public class DatabaseTests : IClassFixture<PostgresFixture>
|
||||
{
|
||||
private readonly PostgresFixture _fixture;
|
||||
|
||||
public DatabaseTests(PostgresFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
[Fact, Trait("Category", TestCategories.Integration)]
|
||||
public async Task Test_DatabaseOperations()
|
||||
{
|
||||
// Use _fixture.ConnectionString to connect
|
||||
using var connection = new NpgsqlConnection(_fixture.ConnectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
// Run migrations
|
||||
await _fixture.RunMigrationsAsync(connection);
|
||||
|
||||
// Test database operations
|
||||
var result = await connection.QueryAsync("SELECT version()");
|
||||
Assert.NotEmpty(result);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**API Reference:**
|
||||
- `PostgresFixture` - xUnit class fixture
|
||||
- `ConnectionString` - PostgreSQL connection string
|
||||
- `RunMigrationsAsync(DbConnection)` - Apply migrations
|
||||
- Requires Docker running locally
|
||||
|
||||
---
|
||||
|
||||
### 6. Valkey Fixture
|
||||
|
||||
Redis-compatible caching for integration tests:
|
||||
|
||||
```csharp
|
||||
using StellaOps.TestKit.Fixtures;
|
||||
|
||||
public class CacheTests : IClassFixture<ValkeyFixture>
|
||||
{
|
||||
private readonly ValkeyFixture _fixture;
|
||||
|
||||
[Fact, Trait("Category", TestCategories.Integration)]
|
||||
public async Task Test_CachingLogic()
|
||||
{
|
||||
var connection = await ConnectionMultiplexer.Connect(_fixture.ConnectionString);
|
||||
var db = connection.GetDatabase();
|
||||
|
||||
await db.StringSetAsync("key", "value");
|
||||
var result = await db.StringGetAsync("key");
|
||||
|
||||
Assert.Equal("value", result.ToString());
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**API Reference:**
|
||||
- `ValkeyFixture` - xUnit class fixture
|
||||
- `ConnectionString` - Redis connection string (host:port)
|
||||
- `Host`, `Port` - Connection details
|
||||
- Uses `redis:7-alpine` image (Valkey-compatible)
|
||||
|
||||
---
|
||||
|
||||
### 7. HTTP Fixture Server
|
||||
|
||||
In-memory API contract testing:
|
||||
|
||||
```csharp
|
||||
using StellaOps.TestKit.Fixtures;
|
||||
|
||||
public class ApiTests : IClassFixture<HttpFixtureServer<Program>>
|
||||
{
|
||||
private readonly HttpClient _client;
|
||||
|
||||
public ApiTests(HttpFixtureServer<Program> fixture)
|
||||
{
|
||||
_client = fixture.CreateClient();
|
||||
}
|
||||
|
||||
[Fact, Trait("Category", TestCategories.Contract)]
|
||||
public async Task Test_HealthEndpoint()
|
||||
{
|
||||
var response = await _client.GetAsync("/health");
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var body = await response.Content.ReadAsStringAsync();
|
||||
Assert.Contains("healthy", body);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**HTTP Message Handler Stub (Hermetic Tests):**
|
||||
|
||||
```csharp
|
||||
[Fact]
|
||||
public async Task Test_ExternalApiCall()
|
||||
{
|
||||
var handler = new HttpMessageHandlerStub()
|
||||
.WhenRequest("https://api.example.com/data", HttpStatusCode.OK, "{\"status\":\"ok\"}");
|
||||
|
||||
var httpClient = new HttpClient(handler);
|
||||
var response = await httpClient.GetAsync("https://api.example.com/data");
|
||||
|
||||
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
||||
}
|
||||
```
|
||||
|
||||
**API Reference:**
|
||||
- `HttpFixtureServer<TProgram>` - WebApplicationFactory wrapper
|
||||
- `CreateClient()` - Get HttpClient for test server
|
||||
- `HttpMessageHandlerStub` - Stub external HTTP dependencies
|
||||
- `WhenRequest(url, statusCode, content)` - Configure stub responses
|
||||
|
||||
---
|
||||
|
||||
### 8. OpenTelemetry Capture
|
||||
|
||||
Trace and span assertion helpers:
|
||||
|
||||
```csharp
|
||||
using StellaOps.TestKit.Observability;
|
||||
|
||||
[Fact]
|
||||
public async Task Test_TracingBehavior()
|
||||
{
|
||||
using var capture = new OtelCapture();
|
||||
|
||||
// Execute code that emits traces
|
||||
await MyService.DoWorkAsync();
|
||||
|
||||
// Assert traces
|
||||
capture.AssertHasSpan("MyService.DoWork");
|
||||
capture.AssertHasTag("user_id", "123");
|
||||
capture.AssertSpanCount(expectedCount: 3);
|
||||
|
||||
// Verify parent-child hierarchy
|
||||
capture.AssertHierarchy("ParentSpan", "ChildSpan");
|
||||
}
|
||||
```
|
||||
|
||||
**API Reference:**
|
||||
- `OtelCapture(string? activitySourceName = null)` - Create capture
|
||||
- `AssertHasSpan(string spanName)` - Verify span exists
|
||||
- `AssertHasTag(string tagKey, string expectedValue)` - Verify tag
|
||||
- `AssertSpanCount(int expectedCount)` - Verify span count
|
||||
- `AssertHierarchy(string parentSpanName, string childSpanName)` - Verify parent-child
|
||||
- `CapturedActivities` - Get all captured spans
|
||||
|
||||
---
|
||||
|
||||
### 9. Test Categories
|
||||
|
||||
Standardized trait constants for CI lane filtering:
|
||||
|
||||
```csharp
|
||||
using StellaOps.TestKit;
|
||||
|
||||
[Fact, Trait("Category", TestCategories.Unit)]
|
||||
public void FastUnitTest() { }
|
||||
|
||||
[Fact, Trait("Category", TestCategories.Integration)]
|
||||
public async Task SlowIntegrationTest() { }
|
||||
|
||||
[Fact, Trait("Category", TestCategories.Live)]
|
||||
public async Task RequiresExternalServices() { }
|
||||
```
|
||||
|
||||
**CI Lane Filtering:**
|
||||
|
||||
```bash
|
||||
# Run only unit tests (fast, no dependencies)
|
||||
dotnet test --filter "Category=Unit"
|
||||
|
||||
# Run all tests except Live
|
||||
dotnet test --filter "Category!=Live"
|
||||
|
||||
# Run Integration + Contract tests
|
||||
dotnet test --filter "Category=Integration|Category=Contract"
|
||||
```
|
||||
|
||||
**Available Categories:**
|
||||
- `Unit` - Fast, in-memory, no external dependencies
|
||||
- `Property` - FsCheck/generative testing
|
||||
- `Snapshot` - Golden master regression
|
||||
- `Integration` - Testcontainers (PostgreSQL, Valkey)
|
||||
- `Contract` - API/WebService contract tests
|
||||
- `Security` - Cryptographic validation
|
||||
- `Performance` - Benchmarking, load tests
|
||||
- `Live` - Requires external services (disabled in CI by default)
|
||||
|
||||
---
|
||||
|
||||
## Best Practices
|
||||
|
||||
### 1. Always Use TestCategories
|
||||
|
||||
Tag every test with the appropriate category:
|
||||
|
||||
```csharp
|
||||
[Fact, Trait("Category", TestCategories.Unit)]
|
||||
public void MyUnitTest() { }
|
||||
```
|
||||
|
||||
This enables CI lane filtering and improves test discoverability.
|
||||
|
||||
### 2. Prefer Deterministic Primitives
|
||||
|
||||
Avoid `DateTime.UtcNow`, `Guid.NewGuid()`, `Random` in tests. Use TestKit alternatives:
|
||||
|
||||
```csharp
|
||||
// ❌ Flaky test (time-dependent)
|
||||
var expiration = DateTime.UtcNow.AddHours(1);
|
||||
|
||||
// ✅ Deterministic test
|
||||
using var time = new DeterministicTime(DateTime.UtcNow);
|
||||
var expiration = time.UtcNow.AddHours(1);
|
||||
```
|
||||
|
||||
### 3. Use Snapshot Tests for Complex Outputs
|
||||
|
||||
For large JSON outputs (SBOM, VEX, attestations), snapshot testing is more maintainable than manual assertions:
|
||||
|
||||
```csharp
|
||||
// ❌ Brittle manual assertions
|
||||
Assert.Equal("SPDX-3.0.1", sbom.SpdxVersion);
|
||||
Assert.Equal(42, sbom.Packages.Count);
|
||||
// ...hundreds of assertions...
|
||||
|
||||
// ✅ Snapshot testing
|
||||
SnapshotAssert.MatchesSnapshot(sbom, "MySbomSnapshot");
|
||||
```
|
||||
|
||||
### 4. Isolate Integration Tests
|
||||
|
||||
Use TestCategories to separate fast unit tests from slow integration tests:
|
||||
|
||||
```csharp
|
||||
[Fact, Trait("Category", TestCategories.Unit)]
|
||||
public void FastTest() { /* no external dependencies */ }
|
||||
|
||||
[Fact, Trait("Category", TestCategories.Integration)]
|
||||
public async Task SlowTest() { /* uses PostgresFixture */ }
|
||||
```
|
||||
|
||||
In CI, run Unit tests first for fast feedback, then Integration tests in parallel.
|
||||
|
||||
### 5. Document Snapshot Baselines
|
||||
|
||||
When updating snapshots (`UPDATE_SNAPSHOTS=1`), add a commit message explaining why:
|
||||
|
||||
```bash
|
||||
git commit -m "Update SBOM snapshot: added new package metadata fields"
|
||||
```
|
||||
|
||||
This helps reviewers understand intentional vs. accidental changes.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Snapshot Mismatch
|
||||
|
||||
**Error:** `Snapshot 'MySbomSnapshot' does not match expected.`
|
||||
|
||||
**Solution:**
|
||||
1. Review diff manually (check `Snapshots/MySbomSnapshot.json`)
|
||||
2. If change is intentional: `UPDATE_SNAPSHOTS=1 dotnet test`
|
||||
3. Commit updated snapshot with explanation
|
||||
|
||||
### Testcontainers Failure
|
||||
|
||||
**Error:** `Docker daemon not running`
|
||||
|
||||
**Solution:**
|
||||
- Ensure Docker Desktop is running
|
||||
- Verify `docker ps` works in terminal
|
||||
- Check Testcontainers logs: `TESTCONTAINERS_DEBUG=1 dotnet test`
|
||||
|
||||
### Determinism Failure
|
||||
|
||||
**Error:** `CanonicalJsonAssert.IsDeterministic failed: byte arrays differ`
|
||||
|
||||
**Root Cause:** Non-deterministic data in serialization (e.g., random GUIDs, timestamps)
|
||||
|
||||
**Solution:**
|
||||
- Use `DeterministicTime` and `DeterministicRandom`
|
||||
- Ensure all data is seeded or mocked
|
||||
- Check for `DateTime.UtcNow` or `Guid.NewGuid()` calls
|
||||
|
||||
---
|
||||
|
||||
## Migration Guide (Existing Tests)
|
||||
|
||||
### Step 1: Add TestKit Reference
|
||||
|
||||
```xml
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.TestKit/StellaOps.TestKit.csproj" />
|
||||
```
|
||||
|
||||
### Step 2: Replace Time-Dependent Code
|
||||
|
||||
**Before:**
|
||||
```csharp
|
||||
var now = DateTime.UtcNow;
|
||||
```
|
||||
|
||||
**After:**
|
||||
```csharp
|
||||
using var time = new DeterministicTime(DateTime.UtcNow);
|
||||
var now = time.UtcNow;
|
||||
```
|
||||
|
||||
### Step 3: Add Test Categories
|
||||
|
||||
```csharp
|
||||
[Fact] // Old
|
||||
[Fact, Trait("Category", TestCategories.Unit)] // New
|
||||
```
|
||||
|
||||
### Step 4: Adopt Snapshot Testing (Optional)
|
||||
|
||||
For complex JSON assertions, replace manual checks with snapshots:
|
||||
|
||||
```csharp
|
||||
// Old
|
||||
Assert.Equal(expected.SpdxVersion, actual.SpdxVersion);
|
||||
// ...
|
||||
|
||||
// New
|
||||
SnapshotAssert.MatchesSnapshot(actual, "TestName");
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## CI Integration
|
||||
|
||||
### Example `.gitea/workflows/test.yml`
|
||||
|
||||
```yaml
|
||||
name: Test Suite
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
unit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-dotnet@v4
|
||||
with:
|
||||
dotnet-version: '10.0.x'
|
||||
- name: Unit Tests (Fast)
|
||||
run: dotnet test --filter "Category=Unit" --logger "trx;LogFileName=unit-results.trx"
|
||||
- name: Upload Results
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: unit-test-results
|
||||
path: '**/unit-results.trx'
|
||||
|
||||
integration:
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
docker:
|
||||
image: docker:dind
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Integration Tests
|
||||
run: dotnet test --filter "Category=Integration" --logger "trx;LogFileName=integration-results.trx"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Support and Feedback
|
||||
|
||||
- **Issues:** Report bugs in sprint tracking files under `docs/implplan/`
|
||||
- **Questions:** Contact Platform Guild
|
||||
- **Documentation:** `src/__Libraries/StellaOps.TestKit/README.md`
|
||||
|
||||
---
|
||||
|
||||
## Changelog
|
||||
|
||||
### v1.0 (2025-12-23)
|
||||
- Initial release: DeterministicTime, DeterministicRandom
|
||||
- CanonicalJsonAssert, SnapshotAssert
|
||||
- PostgresFixture, ValkeyFixture, HttpFixtureServer
|
||||
- OtelCapture for OpenTelemetry traces
|
||||
- TestCategories for CI lane filtering
|
||||
- Pilot adoption in Scanner.Core.Tests
|
||||
Reference in New Issue
Block a user