Add property-based tests for SBOM/VEX document ordering and Unicode normalization determinism

- Implement `SbomVexOrderingDeterminismProperties` for testing component list and vulnerability metadata hash consistency.
- Create `UnicodeNormalizationDeterminismProperties` to validate NFC normalization and Unicode string handling.
- Add project file for `StellaOps.Testing.Determinism.Properties` with necessary dependencies.
- Introduce CI/CD template validation tests including YAML syntax checks and documentation content verification.
- Create validation script for CI/CD templates ensuring all required files and structures are present.
This commit is contained in:
StellaOps Bot
2025-12-26 15:17:15 +02:00
parent 7792749bb4
commit 907783f625
354 changed files with 79727 additions and 1346 deletions

View File

@@ -0,0 +1,255 @@
// -----------------------------------------------------------------------------
// FeedSnapshotCoordinatorTests.cs
// Sprint: SPRINT_20251226_007_BE_determinism_gaps
// Task: DET-GAP-02
// Description: Tests for feed snapshot coordinator determinism
// -----------------------------------------------------------------------------
using StellaOps.Replay.Core.FeedSnapshot;
using Xunit;
namespace StellaOps.Replay.Core.Tests.FeedSnapshot;
public sealed class FeedSnapshotCoordinatorTests
{
[Fact]
public async Task CreateSnapshot_WithMultipleSources_ProducesConsistentDigest()
{
// Arrange
var providers = new IFeedSourceProvider[]
{
new FakeSourceProvider("nvd", "v1", "sha256:abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1", 100),
new FakeSourceProvider("ghsa", "v2", "sha256:def456def456def456def456def456def456def456def456def456def456def4", 200),
new FakeSourceProvider("osv", "v3", "sha256:789012789012789012789012789012789012789012789012789012789012789a", 150)
};
var store = new InMemorySnapshotStore();
var coordinator = new FeedSnapshotCoordinatorService(providers, store);
// Act
var snapshot1 = await coordinator.CreateSnapshotAsync("test-label");
var snapshot2 = await coordinator.CreateSnapshotAsync("test-label");
// Assert - same providers should produce same composite digest
Assert.Equal(snapshot1.CompositeDigest, snapshot2.CompositeDigest);
Assert.Equal(3, snapshot1.Sources.Count);
}
[Fact]
public async Task CreateSnapshot_SourcesAreSortedAlphabetically()
{
// Arrange - providers added in non-alphabetical order
var providers = new IFeedSourceProvider[]
{
new FakeSourceProvider("zebra", "v1", "sha256:aaa1aaa1aaa1aaa1aaa1aaa1aaa1aaa1aaa1aaa1aaa1aaa1aaa1aaa1aaa1aaa1", 10),
new FakeSourceProvider("alpha", "v2", "sha256:bbb2bbb2bbb2bbb2bbb2bbb2bbb2bbb2bbb2bbb2bbb2bbb2bbb2bbb2bbb2bbb2", 20),
new FakeSourceProvider("middle", "v3", "sha256:ccc3ccc3ccc3ccc3ccc3ccc3ccc3ccc3ccc3ccc3ccc3ccc3ccc3ccc3ccc3ccc3", 30)
};
var store = new InMemorySnapshotStore();
var coordinator = new FeedSnapshotCoordinatorService(providers, store);
// Act
var snapshot = await coordinator.CreateSnapshotAsync();
// Assert - sources should be sorted alphabetically
Assert.Equal("alpha", snapshot.Sources[0].SourceId);
Assert.Equal("middle", snapshot.Sources[1].SourceId);
Assert.Equal("zebra", snapshot.Sources[2].SourceId);
}
[Fact]
public async Task CreateSnapshot_WithSubsetOfSources_IncludesOnlyRequested()
{
// Arrange
var providers = new IFeedSourceProvider[]
{
new FakeSourceProvider("nvd", "v1", "sha256:abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1", 100),
new FakeSourceProvider("ghsa", "v2", "sha256:def456def456def456def456def456def456def456def456def456def456def4", 200),
new FakeSourceProvider("osv", "v3", "sha256:789012789012789012789012789012789012789012789012789012789012789a", 150)
};
var store = new InMemorySnapshotStore();
var coordinator = new FeedSnapshotCoordinatorService(providers, store);
// Act
var snapshot = await coordinator.CreateSnapshotAsync(["nvd", "osv"]);
// Assert
Assert.Equal(2, snapshot.Sources.Count);
Assert.Contains(snapshot.Sources, s => s.SourceId == "nvd");
Assert.Contains(snapshot.Sources, s => s.SourceId == "osv");
Assert.DoesNotContain(snapshot.Sources, s => s.SourceId == "ghsa");
}
[Fact]
public async Task RegisteredSources_ReturnsSortedList()
{
// Arrange
var providers = new IFeedSourceProvider[]
{
new FakeSourceProvider("zebra", "v1", "sha256:a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", 10),
new FakeSourceProvider("alpha", "v2", "sha256:b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2", 20)
};
var store = new InMemorySnapshotStore();
var coordinator = new FeedSnapshotCoordinatorService(providers, store);
// Act
var registered = coordinator.RegisteredSources;
// Assert
Assert.Equal(2, registered.Count);
Assert.Equal("alpha", registered[0]);
Assert.Equal("zebra", registered[1]);
}
[Fact]
public async Task GetSnapshot_ReturnsStoredBundle()
{
// Arrange
var providers = new IFeedSourceProvider[]
{
new FakeSourceProvider("nvd", "v1", "sha256:abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1", 100)
};
var store = new InMemorySnapshotStore();
var coordinator = new FeedSnapshotCoordinatorService(providers, store);
var created = await coordinator.CreateSnapshotAsync("test");
// Act
var retrieved = await coordinator.GetSnapshotAsync(created.CompositeDigest);
// Assert
Assert.NotNull(retrieved);
Assert.Equal(created.SnapshotId, retrieved.SnapshotId);
Assert.Equal(created.CompositeDigest, retrieved.CompositeDigest);
}
[Fact]
public async Task ValidateSnapshot_WhenNoChanges_ReturnsValid()
{
// Arrange
var providers = new IFeedSourceProvider[]
{
new FakeSourceProvider("nvd", "v1", "sha256:abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1", 100)
};
var store = new InMemorySnapshotStore();
var coordinator = new FeedSnapshotCoordinatorService(providers, store);
var snapshot = await coordinator.CreateSnapshotAsync();
// Act
var result = await coordinator.ValidateSnapshotAsync(snapshot.CompositeDigest);
// Assert
Assert.True(result.IsValid);
Assert.Null(result.MissingSources);
Assert.Null(result.DriftedSources);
}
[Fact]
public async Task CreateSnapshot_WithUnknownSource_Throws()
{
// Arrange
var providers = new IFeedSourceProvider[]
{
new FakeSourceProvider("nvd", "v1", "sha256:abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1", 100)
};
var store = new InMemorySnapshotStore();
var coordinator = new FeedSnapshotCoordinatorService(providers, store);
// Act & Assert
await Assert.ThrowsAsync<InvalidOperationException>(() =>
coordinator.CreateSnapshotAsync(["nvd", "unknown-source"]));
}
private sealed class FakeSourceProvider : IFeedSourceProvider
{
private readonly string _version;
private readonly string _digest;
private readonly long _recordCount;
public FakeSourceProvider(string sourceId, string version, string digest, long recordCount)
{
SourceId = sourceId;
_version = version;
_digest = digest;
_recordCount = recordCount;
}
public string SourceId { get; }
public string DisplayName => $"Fake {SourceId}";
public int Priority => 0;
public Task<SourceSnapshot> CreateSnapshotAsync(CancellationToken cancellationToken = default)
{
return Task.FromResult(new SourceSnapshot
{
SourceId = SourceId,
Version = _version,
Digest = _digest,
RecordCount = _recordCount
});
}
public Task<string> GetCurrentDigestAsync(CancellationToken cancellationToken = default) =>
Task.FromResult(_digest);
public Task<long> GetRecordCountAsync(CancellationToken cancellationToken = default) =>
Task.FromResult(_recordCount);
public Task ExportAsync(SourceSnapshot snapshot, Stream outputStream, CancellationToken cancellationToken = default) =>
Task.CompletedTask;
public Task<SourceSnapshot> ImportAsync(Stream inputStream, CancellationToken cancellationToken = default) =>
CreateSnapshotAsync(cancellationToken);
}
private sealed class InMemorySnapshotStore : IFeedSnapshotStore
{
private readonly Dictionary<string, FeedSnapshotBundle> _byDigest = new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<string, FeedSnapshotBundle> _byId = new(StringComparer.OrdinalIgnoreCase);
public Task SaveAsync(FeedSnapshotBundle bundle, CancellationToken cancellationToken = default)
{
_byDigest[bundle.CompositeDigest] = bundle;
_byId[bundle.SnapshotId] = bundle;
return Task.CompletedTask;
}
public Task<FeedSnapshotBundle?> GetByDigestAsync(string compositeDigest, CancellationToken cancellationToken = default) =>
Task.FromResult(_byDigest.GetValueOrDefault(compositeDigest));
public Task<FeedSnapshotBundle?> GetByIdAsync(string snapshotId, CancellationToken cancellationToken = default) =>
Task.FromResult(_byId.GetValueOrDefault(snapshotId));
public async IAsyncEnumerable<FeedSnapshotSummary> ListAsync(
DateTimeOffset? from = null,
DateTimeOffset? to = null,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default)
{
foreach (var bundle in _byDigest.Values.OrderByDescending(b => b.CreatedAt))
{
if (from.HasValue && bundle.CreatedAt < from.Value) continue;
if (to.HasValue && bundle.CreatedAt > to.Value) continue;
yield return new FeedSnapshotSummary
{
SnapshotId = bundle.SnapshotId,
CompositeDigest = bundle.CompositeDigest,
Label = bundle.Label,
CreatedAt = bundle.CreatedAt,
SourceCount = bundle.Sources.Count,
TotalRecordCount = bundle.Sources.Sum(s => s.RecordCount)
};
}
}
public Task<bool> DeleteAsync(string compositeDigest, CancellationToken cancellationToken = default)
{
var existed = _byDigest.Remove(compositeDigest, out var bundle);
if (existed && bundle is not null)
{
_byId.Remove(bundle.SnapshotId);
}
return Task.FromResult(existed);
}
}
}

View File

@@ -0,0 +1,399 @@
// -----------------------------------------------------------------------------
// DeterminismManifestValidatorTests.cs
// Sprint: SPRINT_20251226_007_BE_determinism_gaps
// Task: DET-GAP-10
// Description: Tests for determinism manifest validator
// -----------------------------------------------------------------------------
using StellaOps.Replay.Core.Validation;
using Xunit;
namespace StellaOps.Replay.Core.Tests.Validation;
public sealed class DeterminismManifestValidatorTests
{
private readonly DeterminismManifestValidator _validator = new();
[Fact]
public void Validate_ValidManifest_ReturnsValid()
{
// Arrange
var json = """
{
"schemaVersion": "1.0",
"artifact": {
"type": "sbom",
"name": "alpine-3.18",
"version": "2025-12-26T00:00:00Z",
"format": "SPDX 3.0.1"
},
"canonicalHash": {
"algorithm": "SHA-256",
"value": "abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": [
{"name": "StellaOps.Scanner", "version": "1.0.0"}
]
},
"generatedAt": "2025-12-26T12:00:00Z"
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.True(result.IsValid);
Assert.Empty(result.Errors);
}
[Fact]
public void Validate_MissingRequiredField_ReturnsError()
{
// Arrange - missing "artifact"
var json = """
{
"schemaVersion": "1.0",
"canonicalHash": {
"algorithm": "SHA-256",
"value": "abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": []
},
"generatedAt": "2025-12-26T12:00:00Z"
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.False(result.IsValid);
Assert.Contains(result.Errors, e => e.Path == "artifact");
}
[Fact]
public void Validate_InvalidArtifactType_ReturnsError()
{
// Arrange
var json = """
{
"schemaVersion": "1.0",
"artifact": {
"type": "invalid-type",
"name": "test",
"version": "1.0"
},
"canonicalHash": {
"algorithm": "SHA-256",
"value": "abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": []
},
"generatedAt": "2025-12-26T12:00:00Z"
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.False(result.IsValid);
Assert.Contains(result.Errors, e => e.Path == "artifact.type");
}
[Fact]
public void Validate_InvalidHashAlgorithm_ReturnsError()
{
// Arrange
var json = """
{
"schemaVersion": "1.0",
"artifact": {
"type": "sbom",
"name": "test",
"version": "1.0"
},
"canonicalHash": {
"algorithm": "MD5",
"value": "abc123",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": []
},
"generatedAt": "2025-12-26T12:00:00Z"
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.False(result.IsValid);
Assert.Contains(result.Errors, e => e.Path == "canonicalHash.algorithm");
}
[Fact]
public void Validate_InvalidHashValue_ReturnsError()
{
// Arrange - hash value too short
var json = """
{
"schemaVersion": "1.0",
"artifact": {
"type": "sbom",
"name": "test",
"version": "1.0"
},
"canonicalHash": {
"algorithm": "SHA-256",
"value": "abc123",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": []
},
"generatedAt": "2025-12-26T12:00:00Z"
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.False(result.IsValid);
Assert.Contains(result.Errors, e => e.Path == "canonicalHash.value");
}
[Fact]
public void Validate_UnsupportedSchemaVersion_ReturnsError()
{
// Arrange
var json = """
{
"schemaVersion": "2.0",
"artifact": {
"type": "sbom",
"name": "test",
"version": "1.0"
},
"canonicalHash": {
"algorithm": "SHA-256",
"value": "abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": []
},
"generatedAt": "2025-12-26T12:00:00Z"
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.False(result.IsValid);
Assert.Contains(result.Errors, e => e.Path == "schemaVersion");
}
[Fact]
public void Validate_InvalidTimestamp_ReturnsError()
{
// Arrange
var json = """
{
"schemaVersion": "1.0",
"artifact": {
"type": "sbom",
"name": "test",
"version": "1.0"
},
"canonicalHash": {
"algorithm": "SHA-256",
"value": "abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": []
},
"generatedAt": "not-a-timestamp"
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.False(result.IsValid);
Assert.Contains(result.Errors, e => e.Path == "generatedAt");
}
[Fact]
public void Validate_EmptyComponentsArray_ReturnsWarning()
{
// Arrange
var json = """
{
"schemaVersion": "1.0",
"artifact": {
"type": "verdict",
"name": "test",
"version": "1.0"
},
"canonicalHash": {
"algorithm": "SHA-256",
"value": "abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": []
},
"generatedAt": "2025-12-26T12:00:00Z"
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.True(result.IsValid);
Assert.Contains(result.Warnings, w => w.Path == "toolchain.components");
}
[Fact]
public void Validate_SbomWithoutFormat_ReturnsWarning()
{
// Arrange - sbom without format specified
var json = """
{
"schemaVersion": "1.0",
"artifact": {
"type": "sbom",
"name": "test",
"version": "1.0"
},
"canonicalHash": {
"algorithm": "SHA-256",
"value": "abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": [
{"name": "test", "version": "1.0"}
]
},
"generatedAt": "2025-12-26T12:00:00Z"
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.True(result.IsValid);
Assert.Contains(result.Warnings, w => w.Path == "artifact.format");
}
[Fact]
public void Validate_InvalidJson_ReturnsError()
{
// Arrange
var json = "{ invalid json }";
// Act
var result = _validator.Validate(json);
// Assert
Assert.False(result.IsValid);
Assert.Contains(result.Errors, e => e.Path == "$");
}
[Fact]
public void Validate_WithInputs_ValidatesHashFormats()
{
// Arrange
var json = """
{
"schemaVersion": "1.0",
"artifact": {
"type": "verdict",
"name": "test",
"version": "1.0"
},
"canonicalHash": {
"algorithm": "SHA-256",
"value": "abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": [{"name": "test", "version": "1.0"}]
},
"generatedAt": "2025-12-26T12:00:00Z",
"inputs": {
"feedSnapshotHash": "abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1",
"baseImageDigest": "sha256:def456def456def456def456def456def456def456def456def456def456def4"
}
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.True(result.IsValid);
}
[Fact]
public void Validate_InvalidBaseImageDigest_ReturnsError()
{
// Arrange - missing sha256: prefix
var json = """
{
"schemaVersion": "1.0",
"artifact": {
"type": "verdict",
"name": "test",
"version": "1.0"
},
"canonicalHash": {
"algorithm": "SHA-256",
"value": "abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": [{"name": "test", "version": "1.0"}]
},
"generatedAt": "2025-12-26T12:00:00Z",
"inputs": {
"baseImageDigest": "def456def456def456def456def456def456def456def456def456def456def4"
}
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.False(result.IsValid);
Assert.Contains(result.Errors, e => e.Path == "inputs.baseImageDigest");
}
}