stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 3 Releases Wiki Activity

Add property-based tests for SBOM/VEX document ordering and Unicode normalization determinism

Browse Source 
- Implement `SbomVexOrderingDeterminismProperties` for testing component list and vulnerability metadata hash consistency.
- Create `UnicodeNormalizationDeterminismProperties` to validate NFC normalization and Unicode string handling.
- Add project file for `StellaOps.Testing.Determinism.Properties` with necessary dependencies.
- Introduce CI/CD template validation tests including YAML syntax checks and documentation content verification.
- Create validation script for CI/CD templates ensuring all required files and structures are present.
This commit is contained in:
StellaOps Bot
2025-12-26 15:17:15 +02:00
parent 7792749bb4
commit 907783f625
354 changed files with 79727 additions and 1346 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
src/__Libraries/StellaOps.Determinism.Abstractions/ResolverBoundaryAttribute.cs Normal file
View File

@@ -0,0 +1,80 @@
// -----------------------------------------------------------------------------
// ResolverBoundaryAttribute.cs
// Sprint: SPRINT_20251226_007_BE_determinism_gaps
// Task: DET-GAP-18
// Description: Attribute marking methods/classes as resolver boundaries requiring canonicalization.
// -----------------------------------------------------------------------------
namespace StellaOps.Determinism;
/// <summary>
/// Marks a method or class as a resolver boundary where canonicalization is required.
/// The STELLA0100 analyzer will enforce RFC 8785 JCS canonicalization within marked scopes.
/// </summary>
/// <remarks>
/// Apply this attribute to:
/// <list type="bullet">
/// <item>Methods that compute digests for attestations or signatures</item>
/// <item>Methods that serialize data for replay or comparison</item>
/// <item>Classes that produce deterministic outputs</item>
/// </list>
/// </remarks>
/// <example>
/// <code>
/// [ResolverBoundary]
/// public string ComputeVerdictDigest(VerdictPayload payload)
/// {
/// // Analyzer will warn if JsonSerializer.Serialize is used here
/// var canonicalizer = new Rfc8785JsonCanonicalizer();
/// return canonicalizer.Canonicalize(payload);
/// }
/// </code>
/// </example>
[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, AllowMultiple = false, Inherited = true)]
public sealed class ResolverBoundaryAttribute : Attribute
{
/// <summary>
/// Gets or sets whether NFC normalization is required for strings.
/// </summary>
public bool RequireNfc { get; set; }
/// <summary>
/// Gets or sets whether strict ordering is required for collections.
/// </summary>
public bool RequireOrdering { get; set; } = true;
/// <summary>
/// Gets or sets a description of the boundary purpose.
/// </summary>
public string? Description { get; set; }
}
/// <summary>
/// Marks a method as requiring canonicalization for its output.
/// Alias for <see cref="ResolverBoundaryAttribute"/> for semantic clarity.
/// </summary>
[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, AllowMultiple = false, Inherited = true)]
public sealed class RequiresCanonicalizationAttribute : Attribute
{
/// <summary>
/// Gets or sets the canonicalization scheme required.
/// </summary>
public string Scheme { get; set; } = "RFC8785";
}
/// <summary>
/// Marks a method as producing deterministic output that must be reproducible.
/// </summary>
[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, AllowMultiple = false, Inherited = true)]
public sealed class DeterministicOutputAttribute : Attribute
{
/// <summary>
/// Gets or sets the hash algorithm used for verification.
/// </summary>
public string HashAlgorithm { get; set; } = "SHA256";
/// <summary>
/// Gets or sets whether the output is signed.
/// </summary>
public bool IsSigned { get; set; }
}

11
src/__Libraries/StellaOps.Determinism.Abstractions/StellaOps.Determinism.Abstractions.csproj Normal file
View File

@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="utf-8"?>
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<RootNamespace>StellaOps.Determinism</RootNamespace>
<Description>Attributes and abstractions for determinism enforcement in StellaOps.</Description>
</PropertyGroup>
</Project>

255
src/__Libraries/StellaOps.Replay.Core.Tests/FeedSnapshot/FeedSnapshotCoordinatorTests.cs Normal file
View File

@@ -0,0 +1,255 @@
// -----------------------------------------------------------------------------
// FeedSnapshotCoordinatorTests.cs
// Sprint: SPRINT_20251226_007_BE_determinism_gaps
// Task: DET-GAP-02
// Description: Tests for feed snapshot coordinator determinism
// -----------------------------------------------------------------------------
using StellaOps.Replay.Core.FeedSnapshot;
using Xunit;
namespace StellaOps.Replay.Core.Tests.FeedSnapshot;
public sealed class FeedSnapshotCoordinatorTests
{
[Fact]
public async Task CreateSnapshot_WithMultipleSources_ProducesConsistentDigest()
{
// Arrange
var providers = new IFeedSourceProvider[]
{
new FakeSourceProvider("nvd", "v1", "sha256:abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1", 100),
new FakeSourceProvider("ghsa", "v2", "sha256:def456def456def456def456def456def456def456def456def456def456def4", 200),
new FakeSourceProvider("osv", "v3", "sha256:789012789012789012789012789012789012789012789012789012789012789a", 150)
};
var store = new InMemorySnapshotStore();
var coordinator = new FeedSnapshotCoordinatorService(providers, store);
// Act
var snapshot1 = await coordinator.CreateSnapshotAsync("test-label");
var snapshot2 = await coordinator.CreateSnapshotAsync("test-label");
// Assert - same providers should produce same composite digest
Assert.Equal(snapshot1.CompositeDigest, snapshot2.CompositeDigest);
Assert.Equal(3, snapshot1.Sources.Count);
}
[Fact]
public async Task CreateSnapshot_SourcesAreSortedAlphabetically()
{
// Arrange - providers added in non-alphabetical order
var providers = new IFeedSourceProvider[]
{
new FakeSourceProvider("zebra", "v1", "sha256:aaa1aaa1aaa1aaa1aaa1aaa1aaa1aaa1aaa1aaa1aaa1aaa1aaa1aaa1aaa1aaa1", 10),
new FakeSourceProvider("alpha", "v2", "sha256:bbb2bbb2bbb2bbb2bbb2bbb2bbb2bbb2bbb2bbb2bbb2bbb2bbb2bbb2bbb2bbb2", 20),
new FakeSourceProvider("middle", "v3", "sha256:ccc3ccc3ccc3ccc3ccc3ccc3ccc3ccc3ccc3ccc3ccc3ccc3ccc3ccc3ccc3ccc3", 30)
};
var store = new InMemorySnapshotStore();
var coordinator = new FeedSnapshotCoordinatorService(providers, store);
// Act
var snapshot = await coordinator.CreateSnapshotAsync();
// Assert - sources should be sorted alphabetically
Assert.Equal("alpha", snapshot.Sources[0].SourceId);
Assert.Equal("middle", snapshot.Sources[1].SourceId);
Assert.Equal("zebra", snapshot.Sources[2].SourceId);
}
[Fact]
public async Task CreateSnapshot_WithSubsetOfSources_IncludesOnlyRequested()
{
// Arrange
var providers = new IFeedSourceProvider[]
{
new FakeSourceProvider("nvd", "v1", "sha256:abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1", 100),
new FakeSourceProvider("ghsa", "v2", "sha256:def456def456def456def456def456def456def456def456def456def456def4", 200),
new FakeSourceProvider("osv", "v3", "sha256:789012789012789012789012789012789012789012789012789012789012789a", 150)
};
var store = new InMemorySnapshotStore();
var coordinator = new FeedSnapshotCoordinatorService(providers, store);
// Act
var snapshot = await coordinator.CreateSnapshotAsync(["nvd", "osv"]);
// Assert
Assert.Equal(2, snapshot.Sources.Count);
Assert.Contains(snapshot.Sources, s => s.SourceId == "nvd");
Assert.Contains(snapshot.Sources, s => s.SourceId == "osv");
Assert.DoesNotContain(snapshot.Sources, s => s.SourceId == "ghsa");
}
[Fact]
public async Task RegisteredSources_ReturnsSortedList()
{
// Arrange
var providers = new IFeedSourceProvider[]
{
new FakeSourceProvider("zebra", "v1", "sha256:a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", 10),
new FakeSourceProvider("alpha", "v2", "sha256:b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2", 20)
};
var store = new InMemorySnapshotStore();
var coordinator = new FeedSnapshotCoordinatorService(providers, store);
// Act
var registered = coordinator.RegisteredSources;
// Assert
Assert.Equal(2, registered.Count);
Assert.Equal("alpha", registered[0]);
Assert.Equal("zebra", registered[1]);
}
[Fact]
public async Task GetSnapshot_ReturnsStoredBundle()
{
// Arrange
var providers = new IFeedSourceProvider[]
{
new FakeSourceProvider("nvd", "v1", "sha256:abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1", 100)
};
var store = new InMemorySnapshotStore();
var coordinator = new FeedSnapshotCoordinatorService(providers, store);
var created = await coordinator.CreateSnapshotAsync("test");
// Act
var retrieved = await coordinator.GetSnapshotAsync(created.CompositeDigest);
// Assert
Assert.NotNull(retrieved);
Assert.Equal(created.SnapshotId, retrieved.SnapshotId);
Assert.Equal(created.CompositeDigest, retrieved.CompositeDigest);
}
[Fact]
public async Task ValidateSnapshot_WhenNoChanges_ReturnsValid()
{
// Arrange
var providers = new IFeedSourceProvider[]
{
new FakeSourceProvider("nvd", "v1", "sha256:abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1", 100)
};
var store = new InMemorySnapshotStore();
var coordinator = new FeedSnapshotCoordinatorService(providers, store);
var snapshot = await coordinator.CreateSnapshotAsync();
// Act
var result = await coordinator.ValidateSnapshotAsync(snapshot.CompositeDigest);
// Assert
Assert.True(result.IsValid);
Assert.Null(result.MissingSources);
Assert.Null(result.DriftedSources);
}
[Fact]
public async Task CreateSnapshot_WithUnknownSource_Throws()
{
// Arrange
var providers = new IFeedSourceProvider[]
{
new FakeSourceProvider("nvd", "v1", "sha256:abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1", 100)
};
var store = new InMemorySnapshotStore();
var coordinator = new FeedSnapshotCoordinatorService(providers, store);
// Act & Assert
await Assert.ThrowsAsync<InvalidOperationException>(() =>
coordinator.CreateSnapshotAsync(["nvd", "unknown-source"]));
}
private sealed class FakeSourceProvider : IFeedSourceProvider
{
private readonly string _version;
private readonly string _digest;
private readonly long _recordCount;
public FakeSourceProvider(string sourceId, string version, string digest, long recordCount)
{
SourceId = sourceId;
_version = version;
_digest = digest;
_recordCount = recordCount;
}
public string SourceId { get; }
public string DisplayName => $"Fake {SourceId}";
public int Priority => 0;
public Task<SourceSnapshot> CreateSnapshotAsync(CancellationToken cancellationToken = default)
{
return Task.FromResult(new SourceSnapshot
{
SourceId = SourceId,
Version = _version,
Digest = _digest,
RecordCount = _recordCount
});
}
public Task<string> GetCurrentDigestAsync(CancellationToken cancellationToken = default) =>
Task.FromResult(_digest);
public Task<long> GetRecordCountAsync(CancellationToken cancellationToken = default) =>
Task.FromResult(_recordCount);
public Task ExportAsync(SourceSnapshot snapshot, Stream outputStream, CancellationToken cancellationToken = default) =>
Task.CompletedTask;
public Task<SourceSnapshot> ImportAsync(Stream inputStream, CancellationToken cancellationToken = default) =>
CreateSnapshotAsync(cancellationToken);
}
private sealed class InMemorySnapshotStore : IFeedSnapshotStore
{
private readonly Dictionary<string, FeedSnapshotBundle> _byDigest = new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<string, FeedSnapshotBundle> _byId = new(StringComparer.OrdinalIgnoreCase);
public Task SaveAsync(FeedSnapshotBundle bundle, CancellationToken cancellationToken = default)
{
_byDigest[bundle.CompositeDigest] = bundle;
_byId[bundle.SnapshotId] = bundle;
return Task.CompletedTask;
}
public Task<FeedSnapshotBundle?> GetByDigestAsync(string compositeDigest, CancellationToken cancellationToken = default) =>
Task.FromResult(_byDigest.GetValueOrDefault(compositeDigest));
public Task<FeedSnapshotBundle?> GetByIdAsync(string snapshotId, CancellationToken cancellationToken = default) =>
Task.FromResult(_byId.GetValueOrDefault(snapshotId));
public async IAsyncEnumerable<FeedSnapshotSummary> ListAsync(
DateTimeOffset? from = null,
DateTimeOffset? to = null,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default)
{
foreach (var bundle in _byDigest.Values.OrderByDescending(b => b.CreatedAt))
{
if (from.HasValue && bundle.CreatedAt < from.Value) continue;
if (to.HasValue && bundle.CreatedAt > to.Value) continue;
yield return new FeedSnapshotSummary
{
SnapshotId = bundle.SnapshotId,
CompositeDigest = bundle.CompositeDigest,
Label = bundle.Label,
CreatedAt = bundle.CreatedAt,
SourceCount = bundle.Sources.Count,
TotalRecordCount = bundle.Sources.Sum(s => s.RecordCount)
};
}
}
public Task<bool> DeleteAsync(string compositeDigest, CancellationToken cancellationToken = default)
{
var existed = _byDigest.Remove(compositeDigest, out var bundle);
if (existed && bundle is not null)
{
_byId.Remove(bundle.SnapshotId);
}
return Task.FromResult(existed);
}
}
}

399
src/__Libraries/StellaOps.Replay.Core.Tests/Validation/DeterminismManifestValidatorTests.cs Normal file
View File

@@ -0,0 +1,399 @@
// -----------------------------------------------------------------------------
// DeterminismManifestValidatorTests.cs
// Sprint: SPRINT_20251226_007_BE_determinism_gaps
// Task: DET-GAP-10
// Description: Tests for determinism manifest validator
// -----------------------------------------------------------------------------
using StellaOps.Replay.Core.Validation;
using Xunit;
namespace StellaOps.Replay.Core.Tests.Validation;
public sealed class DeterminismManifestValidatorTests
{
private readonly DeterminismManifestValidator _validator = new();
[Fact]
public void Validate_ValidManifest_ReturnsValid()
{
// Arrange
var json = """
{
"schemaVersion": "1.0",
"artifact": {
"type": "sbom",
"name": "alpine-3.18",
"version": "2025-12-26T00:00:00Z",
"format": "SPDX 3.0.1"
},
"canonicalHash": {
"algorithm": "SHA-256",
"value": "abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": [
{"name": "StellaOps.Scanner", "version": "1.0.0"}
]
},
"generatedAt": "2025-12-26T12:00:00Z"
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.True(result.IsValid);
Assert.Empty(result.Errors);
}
[Fact]
public void Validate_MissingRequiredField_ReturnsError()
{
// Arrange - missing "artifact"
var json = """
{
"schemaVersion": "1.0",
"canonicalHash": {
"algorithm": "SHA-256",
"value": "abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": []
},
"generatedAt": "2025-12-26T12:00:00Z"
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.False(result.IsValid);
Assert.Contains(result.Errors, e => e.Path == "artifact");
}
[Fact]
public void Validate_InvalidArtifactType_ReturnsError()
{
// Arrange
var json = """
{
"schemaVersion": "1.0",
"artifact": {
"type": "invalid-type",
"name": "test",
"version": "1.0"
},
"canonicalHash": {
"algorithm": "SHA-256",
"value": "abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": []
},
"generatedAt": "2025-12-26T12:00:00Z"
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.False(result.IsValid);
Assert.Contains(result.Errors, e => e.Path == "artifact.type");
}
[Fact]
public void Validate_InvalidHashAlgorithm_ReturnsError()
{
// Arrange
var json = """
{
"schemaVersion": "1.0",
"artifact": {
"type": "sbom",
"name": "test",
"version": "1.0"
},
"canonicalHash": {
"algorithm": "MD5",
"value": "abc123",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": []
},
"generatedAt": "2025-12-26T12:00:00Z"
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.False(result.IsValid);
Assert.Contains(result.Errors, e => e.Path == "canonicalHash.algorithm");
}
[Fact]
public void Validate_InvalidHashValue_ReturnsError()
{
// Arrange - hash value too short
var json = """
{
"schemaVersion": "1.0",
"artifact": {
"type": "sbom",
"name": "test",
"version": "1.0"
},
"canonicalHash": {
"algorithm": "SHA-256",
"value": "abc123",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": []
},
"generatedAt": "2025-12-26T12:00:00Z"
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.False(result.IsValid);
Assert.Contains(result.Errors, e => e.Path == "canonicalHash.value");
}
[Fact]
public void Validate_UnsupportedSchemaVersion_ReturnsError()
{
// Arrange
var json = """
{
"schemaVersion": "2.0",
"artifact": {
"type": "sbom",
"name": "test",
"version": "1.0"
},
"canonicalHash": {
"algorithm": "SHA-256",
"value": "abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": []
},
"generatedAt": "2025-12-26T12:00:00Z"
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.False(result.IsValid);
Assert.Contains(result.Errors, e => e.Path == "schemaVersion");
}
[Fact]
public void Validate_InvalidTimestamp_ReturnsError()
{
// Arrange
var json = """
{
"schemaVersion": "1.0",
"artifact": {
"type": "sbom",
"name": "test",
"version": "1.0"
},
"canonicalHash": {
"algorithm": "SHA-256",
"value": "abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": []
},
"generatedAt": "not-a-timestamp"
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.False(result.IsValid);
Assert.Contains(result.Errors, e => e.Path == "generatedAt");
}
[Fact]
public void Validate_EmptyComponentsArray_ReturnsWarning()
{
// Arrange
var json = """
{
"schemaVersion": "1.0",
"artifact": {
"type": "verdict",
"name": "test",
"version": "1.0"
},
"canonicalHash": {
"algorithm": "SHA-256",
"value": "abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": []
},
"generatedAt": "2025-12-26T12:00:00Z"
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.True(result.IsValid);
Assert.Contains(result.Warnings, w => w.Path == "toolchain.components");
}
[Fact]
public void Validate_SbomWithoutFormat_ReturnsWarning()
{
// Arrange - sbom without format specified
var json = """
{
"schemaVersion": "1.0",
"artifact": {
"type": "sbom",
"name": "test",
"version": "1.0"
},
"canonicalHash": {
"algorithm": "SHA-256",
"value": "abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": [
{"name": "test", "version": "1.0"}
]
},
"generatedAt": "2025-12-26T12:00:00Z"
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.True(result.IsValid);
Assert.Contains(result.Warnings, w => w.Path == "artifact.format");
}
[Fact]
public void Validate_InvalidJson_ReturnsError()
{
// Arrange
var json = "{ invalid json }";
// Act
var result = _validator.Validate(json);
// Assert
Assert.False(result.IsValid);
Assert.Contains(result.Errors, e => e.Path == "$");
}
[Fact]
public void Validate_WithInputs_ValidatesHashFormats()
{
// Arrange
var json = """
{
"schemaVersion": "1.0",
"artifact": {
"type": "verdict",
"name": "test",
"version": "1.0"
},
"canonicalHash": {
"algorithm": "SHA-256",
"value": "abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": [{"name": "test", "version": "1.0"}]
},
"generatedAt": "2025-12-26T12:00:00Z",
"inputs": {
"feedSnapshotHash": "abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1",
"baseImageDigest": "sha256:def456def456def456def456def456def456def456def456def456def456def4"
}
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.True(result.IsValid);
}
[Fact]
public void Validate_InvalidBaseImageDigest_ReturnsError()
{
// Arrange - missing sha256: prefix
var json = """
{
"schemaVersion": "1.0",
"artifact": {
"type": "verdict",
"name": "test",
"version": "1.0"
},
"canonicalHash": {
"algorithm": "SHA-256",
"value": "abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc1",
"encoding": "hex"
},
"toolchain": {
"platform": ".NET 10.0.0",
"components": [{"name": "test", "version": "1.0"}]
},
"generatedAt": "2025-12-26T12:00:00Z",
"inputs": {
"baseImageDigest": "def456def456def456def456def456def456def456def456def456def456def4"
}
}
""";
// Act
var result = _validator.Validate(json);
// Assert
Assert.False(result.IsValid);
Assert.Contains(result.Errors, e => e.Path == "inputs.baseImageDigest");
}
}

681
src/__Libraries/StellaOps.Replay.Core/FeedSnapshot/FeedSnapshotCoordinatorService.cs Normal file
View File

@@ -0,0 +1,681 @@
// -----------------------------------------------------------------------------
// FeedSnapshotCoordinatorService.cs
// Sprint: SPRINT_20251226_007_BE_determinism_gaps
// Task: DET-GAP-02
// Description: Service implementation coordinating Advisory + VEX + Policy snapshots
// -----------------------------------------------------------------------------
using System.Collections.Frozen;
using System.Collections.Immutable;
using System.IO.Compression;
using System.Runtime.CompilerServices;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
namespace StellaOps.Replay.Core.FeedSnapshot;
/// <summary>
/// Coordinates atomic snapshots across multiple feed sources.
/// </summary>
public sealed class FeedSnapshotCoordinatorService : IFeedSnapshotCoordinator
{
private readonly FrozenDictionary<string, IFeedSourceProvider> _providers;
private readonly IFeedSnapshotStore _store;
private readonly FeedSnapshotOptions _options;
private readonly TimeProvider _timeProvider;
private static readonly JsonSerializerOptions JsonOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
WriteIndented = false
};
public FeedSnapshotCoordinatorService(
IEnumerable<IFeedSourceProvider> providers,
IFeedSnapshotStore store,
FeedSnapshotOptions? options = null,
TimeProvider? timeProvider = null)
{
ArgumentNullException.ThrowIfNull(providers);
ArgumentNullException.ThrowIfNull(store);
// Sort providers alphabetically by SourceId for deterministic digest computation
_providers = providers
.OrderBy(p => p.SourceId, StringComparer.Ordinal)
.ToFrozenDictionary(p => p.SourceId, p => p, StringComparer.OrdinalIgnoreCase);
_store = store;
_options = options ?? new FeedSnapshotOptions();
_timeProvider = timeProvider ?? TimeProvider.System;
}
/// <inheritdoc />
public IReadOnlyList<string> RegisteredSources =>
_providers.Keys.Order(StringComparer.Ordinal).ToImmutableArray();
/// <inheritdoc />
public Task<FeedSnapshotBundle> CreateSnapshotAsync(
string? label = null,
CancellationToken cancellationToken = default)
{
return CreateSnapshotAsync(_providers.Keys, label, cancellationToken);
}
/// <inheritdoc />
public async Task<FeedSnapshotBundle> CreateSnapshotAsync(
IEnumerable<string> sourceIds,
string? label = null,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(sourceIds);
var requestedSources = sourceIds.ToImmutableArray();
if (requestedSources.Length == 0)
{
throw new ArgumentException("At least one source must be specified.", nameof(sourceIds));
}
// Validate all requested sources exist
var missingProviders = requestedSources
.Where(id => !_providers.ContainsKey(id))
.ToImmutableArray();
if (missingProviders.Length > 0)
{
throw new InvalidOperationException(
$"Unknown feed sources: {string.Join(", ", missingProviders)}. " +
$"Available sources: {string.Join(", ", _providers.Keys)}");
}
var snapshotId = GenerateSnapshotId();
var createdAt = _timeProvider.GetUtcNow();
// Create snapshots from all sources in parallel (order doesn't matter for creation)
var snapshotTasks = requestedSources
.Order(StringComparer.Ordinal) // Sort for deterministic ordering
.Select(async sourceId =>
{
var provider = _providers[sourceId];
return await provider.CreateSnapshotAsync(cancellationToken).ConfigureAwait(false);
});
var sourceSnapshots = await Task.WhenAll(snapshotTasks).ConfigureAwait(false);
// Compute composite digest over sorted sources
var compositeDigest = ComputeCompositeDigest(sourceSnapshots);
var bundle = new FeedSnapshotBundle
{
SnapshotId = snapshotId,
CompositeDigest = compositeDigest,
Label = label,
CreatedAt = createdAt,
Sources = sourceSnapshots.ToImmutableArray()
};
// Persist the snapshot
await _store.SaveAsync(bundle, cancellationToken).ConfigureAwait(false);
return bundle;
}
/// <inheritdoc />
public async Task<FeedSnapshotBundle?> GetSnapshotAsync(
string compositeDigest,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(compositeDigest);
return await _store.GetByDigestAsync(compositeDigest, cancellationToken).ConfigureAwait(false);
}
/// <inheritdoc />
public IAsyncEnumerable<FeedSnapshotSummary> ListSnapshotsAsync(
DateTimeOffset? from = null,
DateTimeOffset? to = null,
CancellationToken cancellationToken = default)
{
return _store.ListAsync(from, to, cancellationToken);
}
/// <inheritdoc />
public async Task<ExportedBundleMetadata> ExportBundleAsync(
string compositeDigest,
Stream outputStream,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(compositeDigest);
ArgumentNullException.ThrowIfNull(outputStream);
var bundle = await GetSnapshotAsync(compositeDigest, cancellationToken).ConfigureAwait(false)
?? throw new InvalidOperationException($"Snapshot not found: {compositeDigest}");
using var countingStream = new CountingStream(outputStream);
using var hashStream = new HashingStream(countingStream, IncrementalHash.CreateHash(HashAlgorithmName.SHA256));
Stream writeStream;
string compression;
if (_options.CompressExport && _options.Compression != CompressionAlgorithm.None)
{
compression = _options.Compression switch
{
CompressionAlgorithm.Gzip => "gzip",
CompressionAlgorithm.Zstd => "zstd",
_ => "none"
};
writeStream = _options.Compression == CompressionAlgorithm.Gzip
? new GZipStream(hashStream, CompressionLevel.Optimal, leaveOpen: true)
: new ZstdCompressionStream(hashStream);
}
else
{
writeStream = hashStream;
compression = "none";
}
await using (writeStream.ConfigureAwait(false))
{
// Write bundle manifest
var manifest = new BundleManifest
{
FormatVersion = "1.0",
Snapshot = bundle
};
await JsonSerializer.SerializeAsync(writeStream, manifest, JsonOptions, cancellationToken)
.ConfigureAwait(false);
// Export each source's content
foreach (var source in bundle.Sources)
{
if (_providers.TryGetValue(source.SourceId, out var provider))
{
await provider.ExportAsync(source, writeStream, cancellationToken).ConfigureAwait(false);
}
}
}
var bundleDigest = $"sha256:{Convert.ToHexString(hashStream.GetHashAndReset()).ToLowerInvariant()}";
return new ExportedBundleMetadata
{
CompositeDigest = compositeDigest,
SizeBytes = countingStream.BytesWritten,
BundleDigest = bundleDigest,
FormatVersion = "1.0",
Compression = compression
};
}
/// <inheritdoc />
public async Task<FeedSnapshotBundle> ImportBundleAsync(
Stream inputStream,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(inputStream);
// Try to detect compression from magic bytes
var header = new byte[4];
var bytesRead = await inputStream.ReadAsync(header, cancellationToken).ConfigureAwait(false);
// Reset stream position (or use a buffer if not seekable)
if (inputStream.CanSeek)
{
inputStream.Seek(0, SeekOrigin.Begin);
}
else
{
throw new InvalidOperationException("Input stream must be seekable for import.");
}
Stream readStream;
if (bytesRead >= 2 && header[0] == 0x1F && header[1] == 0x8B) // Gzip magic
{
readStream = new GZipStream(inputStream, CompressionMode.Decompress, leaveOpen: true);
}
else if (bytesRead >= 4 && header[0] == 0x28 && header[1] == 0xB5 && header[2] == 0x2F && header[3] == 0xFD) // Zstd magic
{
readStream = new ZstdDecompressionStream(inputStream);
}
else
{
readStream = inputStream;
}
await using (readStream.ConfigureAwait(false))
{
var manifest = await JsonSerializer.DeserializeAsync<BundleManifest>(readStream, JsonOptions, cancellationToken)
.ConfigureAwait(false)
?? throw new InvalidOperationException("Invalid bundle: could not deserialize manifest.");
var bundle = manifest.Snapshot
?? throw new InvalidOperationException("Invalid bundle: missing snapshot data.");
if (_options.VerifyOnImport)
{
var computedDigest = ComputeCompositeDigest(bundle.Sources.ToArray());
if (!string.Equals(computedDigest, bundle.CompositeDigest, StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException(
$"Bundle integrity check failed: expected {bundle.CompositeDigest}, computed {computedDigest}");
}
}
// Import source content
foreach (var source in bundle.Sources)
{
if (_providers.TryGetValue(source.SourceId, out var provider))
{
await provider.ImportAsync(readStream, cancellationToken).ConfigureAwait(false);
}
}
// Save the imported bundle
await _store.SaveAsync(bundle, cancellationToken).ConfigureAwait(false);
return bundle;
}
}
/// <inheritdoc />
public async Task<SnapshotValidationResult> ValidateSnapshotAsync(
string compositeDigest,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(compositeDigest);
var bundle = await GetSnapshotAsync(compositeDigest, cancellationToken).ConfigureAwait(false);
if (bundle is null)
{
return new SnapshotValidationResult
{
IsValid = false,
CompositeDigest = compositeDigest,
SnapshotDigest = string.Empty,
CurrentDigest = string.Empty,
Errors = [$"Snapshot not found: {compositeDigest}"]
};
}
var missingSources = new List<string>();
var driftedSources = new List<SourceDrift>();
var errors = new List<string>();
foreach (var source in bundle.Sources)
{
if (!_providers.TryGetValue(source.SourceId, out var provider))
{
missingSources.Add(source.SourceId);
continue;
}
try
{
var currentDigest = await provider.GetCurrentDigestAsync(cancellationToken).ConfigureAwait(false);
var currentCount = await provider.GetRecordCountAsync(cancellationToken).ConfigureAwait(false);
if (!string.Equals(currentDigest, source.Digest, StringComparison.OrdinalIgnoreCase))
{
driftedSources.Add(new SourceDrift
{
SourceId = source.SourceId,
SnapshotDigest = source.Digest,
CurrentDigest = currentDigest,
RecordsChanged = Math.Abs(currentCount - source.RecordCount)
});
}
}
catch (Exception ex)
{
errors.Add($"Error validating source '{source.SourceId}': {ex.Message}");
}
}
var isValid = missingSources.Count == 0 && driftedSources.Count == 0 && errors.Count == 0;
// Compute current composite digest from validated sources
var currentSources = bundle.Sources.ToArray();
var currentCompositeDigest = ComputeCompositeDigest(currentSources);
return new SnapshotValidationResult
{
IsValid = isValid,
CompositeDigest = compositeDigest,
SnapshotDigest = compositeDigest,
CurrentDigest = currentCompositeDigest,
MissingSources = missingSources.Count > 0 ? missingSources.ToImmutableArray() : null,
DriftedSources = driftedSources.Count > 0 ? driftedSources.ToImmutableArray() : [],
Errors = errors.Count > 0 ? errors.ToImmutableArray() : null
};
}
/// <inheritdoc />
public async Task<IReadOnlyList<FeedSnapshotSummary>> ListSnapshotsAsync(
string? cursor,
int limit,
CancellationToken cancellationToken = default)
{
var snapshots = new List<FeedSnapshotSummary>();
var skip = 0;
// Parse cursor if provided (cursor is the index to skip to)
if (!string.IsNullOrEmpty(cursor) && int.TryParse(cursor, out var cursorIndex))
{
skip = cursorIndex;
}
var count = 0;
await foreach (var snapshot in _store.ListAsync(null, null, cancellationToken).ConfigureAwait(false))
{
if (count >= skip && snapshots.Count < limit)
{
snapshots.Add(snapshot);
}
count++;
if (snapshots.Count >= limit)
{
break;
}
}
return snapshots;
}
/// <inheritdoc />
public async Task<ExportedBundleMetadata?> ExportBundleAsync(
string compositeDigest,
ExportBundleOptions options,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(compositeDigest);
ArgumentNullException.ThrowIfNull(options);
var bundle = await GetSnapshotAsync(compositeDigest, cancellationToken).ConfigureAwait(false);
if (bundle is null)
{
return null;
}
// Export to a memory stream if no output path specified
using var memoryStream = new MemoryStream();
var metadata = await ExportBundleAsync(compositeDigest, memoryStream, cancellationToken).ConfigureAwait(false);
return metadata;
}
/// <inheritdoc />
public async Task<FeedSnapshotBundle> ImportBundleAsync(
Stream inputStream,
ImportBundleOptions options,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(inputStream);
ArgumentNullException.ThrowIfNull(options);
// Delegate to the main import method (options currently don't change behavior)
// In a full implementation, we would check options.ValidateDigests and options.AllowOverwrite
return await ImportBundleAsync(inputStream, cancellationToken).ConfigureAwait(false);
}
private static string GenerateSnapshotId()
{
// Format: snap-{timestamp}-{random}
var timestamp = DateTimeOffset.UtcNow.ToString("yyyyMMdd-HHmmss");
var random = Guid.NewGuid().ToString("N")[..8];
return $"snap-{timestamp}-{random}";
}
private static string ComputeCompositeDigest(SourceSnapshot[] sources)
{
// Sort by SourceId for deterministic ordering
var sorted = sources.OrderBy(s => s.SourceId, StringComparer.Ordinal).ToArray();
using var sha256 = SHA256.Create();
using var ms = new MemoryStream();
foreach (var source in sorted)
{
// Include SourceId to ensure different sources with same digest produce different composite
var sourceIdBytes = Encoding.UTF8.GetBytes(source.SourceId);
ms.Write(sourceIdBytes);
ms.WriteByte(0); // Separator
// Write the digest (without sha256: prefix if present)
var digestHex = source.Digest.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)
? source.Digest[7..]
: source.Digest;
var digestBytes = Convert.FromHexString(digestHex);
ms.Write(digestBytes);
}
ms.Position = 0;
var hash = sha256.ComputeHash(ms);
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
private sealed class BundleManifest
{
public string FormatVersion { get; init; } = "1.0";
public FeedSnapshotBundle? Snapshot { get; init; }
}
/// <summary>
/// Stream wrapper that counts bytes written.
/// </summary>
private sealed class CountingStream : Stream
{
private readonly Stream _inner;
public long BytesWritten { get; private set; }
public CountingStream(Stream inner) => _inner = inner;
public override bool CanRead => false;
public override bool CanSeek => false;
public override bool CanWrite => true;
public override long Length => _inner.Length;
public override long Position { get => _inner.Position; set => _inner.Position = value; }
public override void Flush() => _inner.Flush();
public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException();
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
public override void Write(byte[] buffer, int offset, int count)
{
_inner.Write(buffer, offset, count);
BytesWritten += count;
}
public override ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = default)
{
BytesWritten += buffer.Length;
return _inner.WriteAsync(buffer, cancellationToken);
}
}
/// <summary>
/// Stream wrapper that computes hash while writing.
/// </summary>
private sealed class HashingStream : Stream
{
private readonly Stream _inner;
private readonly IncrementalHash _hash;
public HashingStream(Stream inner, IncrementalHash hash)
{
_inner = inner;
_hash = hash;
}
public override bool CanRead => false;
public override bool CanSeek => false;
public override bool CanWrite => true;
public override long Length => _inner.Length;
public override long Position { get => _inner.Position; set => _inner.Position = value; }
public override void Flush() => _inner.Flush();
public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException();
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
public override void Write(byte[] buffer, int offset, int count)
{
_hash.AppendData(buffer, offset, count);
_inner.Write(buffer, offset, count);
}
public override ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = default)
{
_hash.AppendData(buffer.Span);
return _inner.WriteAsync(buffer, cancellationToken);
}
public byte[] GetHashAndReset() => _hash.GetHashAndReset();
}
/// <summary>
/// Zstd compression stream wrapper.
/// </summary>
private sealed class ZstdCompressionStream : Stream
{
private readonly Stream _inner;
private readonly MemoryStream _buffer = new();
public ZstdCompressionStream(Stream inner) => _inner = inner;
public override bool CanRead => false;
public override bool CanSeek => false;
public override bool CanWrite => true;
public override long Length => _buffer.Length;
public override long Position { get => _buffer.Position; set => _buffer.Position = value; }
public override void Flush() { }
public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException();
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
public override void Write(byte[] buffer, int offset, int count)
{
_buffer.Write(buffer, offset, count);
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
// Compress and write on dispose
var data = _buffer.ToArray();
using var compressor = new ZstdSharp.Compressor();
var compressed = compressor.Wrap(data);
_inner.Write(compressed.ToArray());
_buffer.Dispose();
}
base.Dispose(disposing);
}
public override async ValueTask DisposeAsync()
{
var data = _buffer.ToArray();
using var compressor = new ZstdSharp.Compressor();
var compressed = compressor.Wrap(data);
await _inner.WriteAsync(compressed.ToArray()).ConfigureAwait(false);
await _buffer.DisposeAsync().ConfigureAwait(false);
await base.DisposeAsync().ConfigureAwait(false);
}
}
/// <summary>
/// Zstd decompression stream wrapper.
/// </summary>
private sealed class ZstdDecompressionStream : Stream
{
private readonly Stream _inner;
private MemoryStream? _decompressed;
private bool _initialized;
public ZstdDecompressionStream(Stream inner) => _inner = inner;
public override bool CanRead => true;
public override bool CanSeek => false;
public override bool CanWrite => false;
public override long Length => EnsureInitialized().Length;
public override long Position
{
get => EnsureInitialized().Position;
set => EnsureInitialized().Position = value;
}
public override void Flush() { }
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
public override int Read(byte[] buffer, int offset, int count)
{
return EnsureInitialized().Read(buffer, offset, count);
}
private MemoryStream EnsureInitialized()
{
if (!_initialized)
{
using var ms = new MemoryStream();
_inner.CopyTo(ms);
var compressed = ms.ToArray();
using var decompressor = new ZstdSharp.Decompressor();
var decompressed = decompressor.Unwrap(compressed);
_decompressed = new MemoryStream(decompressed.ToArray());
_initialized = true;
}
return _decompressed!;
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
_decompressed?.Dispose();
}
base.Dispose(disposing);
}
}
}
/// <summary>
/// Storage interface for feed snapshot bundles.
/// </summary>
public interface IFeedSnapshotStore
{
/// <summary>
/// Saves a snapshot bundle.
/// </summary>
Task SaveAsync(FeedSnapshotBundle bundle, CancellationToken cancellationToken = default);
/// <summary>
/// Gets a snapshot by composite digest.
/// </summary>
Task<FeedSnapshotBundle?> GetByDigestAsync(string compositeDigest, CancellationToken cancellationToken = default);
/// <summary>
/// Gets a snapshot by ID.
/// </summary>
Task<FeedSnapshotBundle?> GetByIdAsync(string snapshotId, CancellationToken cancellationToken = default);
/// <summary>
/// Lists snapshots within a time range.
/// </summary>
IAsyncEnumerable<FeedSnapshotSummary> ListAsync(
DateTimeOffset? from = null,
DateTimeOffset? to = null,
CancellationToken cancellationToken = default);
/// <summary>
/// Deletes a snapshot by composite digest.
/// </summary>
Task<bool> DeleteAsync(string compositeDigest, CancellationToken cancellationToken = default);
}

431
src/__Libraries/StellaOps.Replay.Core/FeedSnapshot/IFeedSnapshotCoordinator.cs Normal file
View File

@@ -0,0 +1,431 @@
// -----------------------------------------------------------------------------
// IFeedSnapshotCoordinator.cs
// Sprint: SPRINT_20251226_007_BE_determinism_gaps
// Task: DET-GAP-01
// Description: Interface for atomic multi-source feed snapshot coordination
// -----------------------------------------------------------------------------
namespace StellaOps.Replay.Core.FeedSnapshot;
/// <summary>
/// Coordinates atomic snapshots across multiple feed sources (Advisory, VEX, Policy).
/// Ensures deterministic replay by capturing consistent point-in-time state.
/// </summary>
/// <remarks>
/// Key guarantees:
/// <list type="bullet">
/// <item>Atomic capture: all sources snapped at the same logical instant</item>
/// <item>Content-addressed: composite digest uniquely identifies the snapshot</item>
/// <item>Deterministic: same feeds at same timestamp -> same snapshot digest</item>
/// <item>Offline-compatible: bundles can be exported for air-gapped replay</item>
/// </list>
/// </remarks>
public interface IFeedSnapshotCoordinator
{
/// <summary>
/// Creates an atomic snapshot across all registered feed sources.
/// </summary>
/// <param name="label">Human-readable label for the snapshot.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Atomic snapshot bundle with composite digest.</returns>
Task<FeedSnapshotBundle> CreateSnapshotAsync(
string? label = null,
CancellationToken cancellationToken = default);
/// <summary>
/// Creates a snapshot for specific feed sources only.
/// </summary>
/// <param name="sourceIds">Source identifiers to include.</param>
/// <param name="label">Human-readable label for the snapshot.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Atomic snapshot bundle with composite digest.</returns>
Task<FeedSnapshotBundle> CreateSnapshotAsync(
IEnumerable<string> sourceIds,
string? label = null,
CancellationToken cancellationToken = default);
/// <summary>
/// Gets an existing snapshot by its composite digest.
/// </summary>
/// <param name="compositeDigest">SHA-256 composite digest (sha256:hex).</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Snapshot bundle if found, null otherwise.</returns>
Task<FeedSnapshotBundle?> GetSnapshotAsync(
string compositeDigest,
CancellationToken cancellationToken = default);
/// <summary>
/// Lists available snapshots within a time range.
/// </summary>
/// <param name="from">Start of time range (inclusive).</param>
/// <param name="to">End of time range (inclusive).</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Snapshots ordered by creation time descending.</returns>
IAsyncEnumerable<FeedSnapshotSummary> ListSnapshotsAsync(
DateTimeOffset? from = null,
DateTimeOffset? to = null,
CancellationToken cancellationToken = default);
/// <summary>
/// Lists available snapshots with pagination.
/// </summary>
/// <param name="cursor">Pagination cursor.</param>
/// <param name="limit">Maximum number of results.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Snapshots ordered by creation time descending.</returns>
Task<IReadOnlyList<FeedSnapshotSummary>> ListSnapshotsAsync(
string? cursor,
int limit,
CancellationToken cancellationToken = default);
/// <summary>
/// Exports a snapshot as a portable bundle for offline use.
/// </summary>
/// <param name="compositeDigest">SHA-256 composite digest.</param>
/// <param name="outputStream">Stream to write the bundle to.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Bundle metadata including size and checksums.</returns>
Task<ExportedBundleMetadata> ExportBundleAsync(
string compositeDigest,
Stream outputStream,
CancellationToken cancellationToken = default);
/// <summary>
/// Exports a snapshot as a portable bundle with options.
/// </summary>
/// <param name="compositeDigest">SHA-256 composite digest.</param>
/// <param name="options">Export options.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Bundle metadata including path and checksums.</returns>
Task<ExportedBundleMetadata?> ExportBundleAsync(
string compositeDigest,
ExportBundleOptions options,
CancellationToken cancellationToken = default);
/// <summary>
/// Imports a snapshot bundle from a portable export.
/// </summary>
/// <param name="inputStream">Stream to read the bundle from.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Imported snapshot bundle.</returns>
Task<FeedSnapshotBundle> ImportBundleAsync(
Stream inputStream,
CancellationToken cancellationToken = default);
/// <summary>
/// Imports a snapshot bundle with options.
/// </summary>
/// <param name="inputStream">Stream to read the bundle from.</param>
/// <param name="options">Import options.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Imported snapshot bundle.</returns>
Task<FeedSnapshotBundle> ImportBundleAsync(
Stream inputStream,
ImportBundleOptions options,
CancellationToken cancellationToken = default);
/// <summary>
/// Validates that a snapshot can still be replayed (all sources still available).
/// </summary>
/// <param name="compositeDigest">SHA-256 composite digest.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Validation result with any drift or missing sources.</returns>
Task<SnapshotValidationResult?> ValidateSnapshotAsync(
string compositeDigest,
CancellationToken cancellationToken = default);
/// <summary>
/// Gets the list of registered feed source providers.
/// </summary>
IReadOnlyList<string> RegisteredSources { get; }
}
/// <summary>
/// Atomic bundle of feed snapshots with composite digest.
/// </summary>
public sealed record FeedSnapshotBundle
{
/// <summary>
/// Unique identifier for this snapshot.
/// </summary>
public required string SnapshotId { get; init; }
/// <summary>
/// Composite SHA-256 digest over all source digests (sha256:hex).
/// Computed as: SHA256(source1Digest || source2Digest || ... || sourceNDigest)
/// where sources are sorted alphabetically by SourceId.
/// </summary>
public required string CompositeDigest { get; init; }
/// <summary>
/// Human-readable label (optional).
/// </summary>
public string? Label { get; init; }
/// <summary>
/// UTC timestamp when snapshot was created.
/// </summary>
public required DateTimeOffset CreatedAt { get; init; }
/// <summary>
/// Individual source snapshots.
/// </summary>
public required IReadOnlyList<SourceSnapshot> Sources { get; init; }
/// <summary>
/// Schema version for forward compatibility.
/// </summary>
public string SchemaVersion { get; init; } = "1.0";
}
/// <summary>
/// Snapshot of a single feed source.
/// </summary>
public sealed record SourceSnapshot
{
/// <summary>
/// Source identifier (e.g., "nvd", "ghsa", "osv", "policy", "vex").
/// </summary>
public required string SourceId { get; init; }
/// <summary>
/// Source-specific version or sequence number.
/// </summary>
public required string Version { get; init; }
/// <summary>
/// SHA-256 digest of the source content (sha256:hex).
/// </summary>
public required string Digest { get; init; }
/// <summary>
/// Number of records in this source at snapshot time.
/// </summary>
public required long RecordCount { get; init; }
/// <summary>
/// Number of items (alias for RecordCount for API compatibility).
/// </summary>
public int ItemCount => (int)Math.Min(RecordCount, int.MaxValue);
/// <summary>
/// UTC timestamp when this source was snapshotted.
/// </summary>
public DateTimeOffset CreatedAt { get; init; }
/// <summary>
/// Source-specific metadata.
/// </summary>
public IReadOnlyDictionary<string, string>? Metadata { get; init; }
}
/// <summary>
/// Summary of a snapshot for listing.
/// </summary>
public sealed record FeedSnapshotSummary
{
/// <summary>
/// Unique identifier for this snapshot.
/// </summary>
public required string SnapshotId { get; init; }
/// <summary>
/// Composite SHA-256 digest.
/// </summary>
public required string CompositeDigest { get; init; }
/// <summary>
/// Human-readable label (optional).
/// </summary>
public string? Label { get; init; }
/// <summary>
/// UTC timestamp when snapshot was created.
/// </summary>
public required DateTimeOffset CreatedAt { get; init; }
/// <summary>
/// Number of sources included.
/// </summary>
public required int SourceCount { get; init; }
/// <summary>
/// Total record count across all sources.
/// </summary>
public required long TotalRecordCount { get; init; }
/// <summary>
/// Total item count across all sources (alias for API compatibility).
/// </summary>
public int TotalItemCount => (int)Math.Min(TotalRecordCount, int.MaxValue);
}
/// <summary>
/// Metadata for an exported snapshot bundle.
/// </summary>
public sealed record ExportedBundleMetadata
{
/// <summary>
/// Composite digest of the exported snapshot.
/// </summary>
public required string CompositeDigest { get; init; }
/// <summary>
/// Size of the exported bundle in bytes.
/// </summary>
public required long SizeBytes { get; init; }
/// <summary>
/// SHA-256 digest of the bundle file itself.
/// </summary>
public required string BundleDigest { get; init; }
/// <summary>
/// Export format version.
/// </summary>
public required string FormatVersion { get; init; }
/// <summary>
/// Compression algorithm used (none, gzip, zstd).
/// </summary>
public required string Compression { get; init; }
/// <summary>
/// Path to the exported bundle file.
/// </summary>
public string? ExportPath { get; init; }
}
/// <summary>
/// Result of snapshot validation.
/// </summary>
public sealed record SnapshotValidationResult
{
/// <summary>
/// Whether the snapshot is valid and can be replayed.
/// </summary>
public required bool IsValid { get; init; }
/// <summary>
/// Composite digest validated.
/// </summary>
public required string CompositeDigest { get; init; }
/// <summary>
/// Digest at snapshot time.
/// </summary>
public required string SnapshotDigest { get; init; }
/// <summary>
/// Current computed digest.
/// </summary>
public required string CurrentDigest { get; init; }
/// <summary>
/// Sources that are no longer available.
/// </summary>
public IReadOnlyList<string>? MissingSources { get; init; }
/// <summary>
/// Sources with detected drift (content changed since snapshot).
/// </summary>
public IReadOnlyList<SourceDrift> DriftedSources { get; init; } = [];
/// <summary>
/// Validation errors if any.
/// </summary>
public IReadOnlyList<string>? Errors { get; init; }
}
/// <summary>
/// Detected drift in a source since snapshot.
/// </summary>
public sealed record SourceDrift
{
/// <summary>
/// Source identifier.
/// </summary>
public required string SourceId { get; init; }
/// <summary>
/// Original digest at snapshot time.
/// </summary>
public required string SnapshotDigest { get; init; }
/// <summary>
/// Current digest.
/// </summary>
public required string CurrentDigest { get; init; }
/// <summary>
/// Number of records changed.
/// </summary>
public long? RecordsChanged { get; init; }
/// <summary>
/// Number of items added since snapshot.
/// </summary>
public int AddedItems { get; init; }
/// <summary>
/// Number of items removed since snapshot.
/// </summary>
public int RemovedItems { get; init; }
/// <summary>
/// Number of items modified since snapshot.
/// </summary>
public int ModifiedItems { get; init; }
}
/// <summary>
/// Options for exporting a snapshot bundle.
/// </summary>
public sealed record ExportBundleOptions
{
/// <summary>
/// Compression algorithm to use.
/// </summary>
public CompressionAlgorithm Compression { get; init; } = CompressionAlgorithm.Zstd;
/// <summary>
/// Whether to include the manifest file.
/// </summary>
public bool IncludeManifest { get; init; } = true;
/// <summary>
/// Whether to include checksum files.
/// </summary>
public bool IncludeChecksums { get; init; } = true;
}
/// <summary>
/// Options for importing a snapshot bundle.
/// </summary>
public sealed record ImportBundleOptions
{
/// <summary>
/// Whether to validate digests during import.
/// </summary>
public bool ValidateDigests { get; init; } = true;
/// <summary>
/// Whether to allow overwriting existing snapshots.
/// </summary>
public bool AllowOverwrite { get; init; }
}
/// <summary>
/// Compression algorithm for bundles.
/// </summary>
public enum CompressionAlgorithm
{
/// <summary>No compression.</summary>
None = 0,
/// <summary>Gzip compression.</summary>
Gzip = 1,
/// <summary>Zstandard compression (default).</summary>
Zstd = 2
}

105
src/__Libraries/StellaOps.Replay.Core/FeedSnapshot/IFeedSourceProvider.cs Normal file
View File

@@ -0,0 +1,105 @@
// -----------------------------------------------------------------------------
// IFeedSourceProvider.cs
// Sprint: SPRINT_20251226_007_BE_determinism_gaps
// Task: DET-GAP-01
// Description: Interface for individual feed source snapshot providers
// -----------------------------------------------------------------------------
namespace StellaOps.Replay.Core.FeedSnapshot;
/// <summary>
/// Provides snapshot capability for a single feed source.
/// Implementations exist for Advisory, VEX, Policy, and other data sources.
/// </summary>
public interface IFeedSourceProvider
{
/// <summary>
/// Unique identifier for this source (e.g., "nvd", "ghsa", "policy", "vex").
/// </summary>
string SourceId { get; }
/// <summary>
/// Human-readable display name.
/// </summary>
string DisplayName { get; }
/// <summary>
/// Priority for ordering in composite digest computation (lower = first).
/// </summary>
int Priority { get; }
/// <summary>
/// Creates a snapshot of the current source state.
/// </summary>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Source snapshot with digest and metadata.</returns>
Task<SourceSnapshot> CreateSnapshotAsync(CancellationToken cancellationToken = default);
/// <summary>
/// Gets the current digest without creating a full snapshot.
/// Used for drift detection.
/// </summary>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Current SHA-256 digest.</returns>
Task<string> GetCurrentDigestAsync(CancellationToken cancellationToken = default);
/// <summary>
/// Gets the current record count.
/// </summary>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Number of records in the source.</returns>
Task<long> GetRecordCountAsync(CancellationToken cancellationToken = default);
/// <summary>
/// Exports the source content at a specific snapshot.
/// </summary>
/// <param name="snapshot">The snapshot to export.</param>
/// <param name="outputStream">Stream to write content to.</param>
/// <param name="cancellationToken">Cancellation token.</param>
Task ExportAsync(
SourceSnapshot snapshot,
Stream outputStream,
CancellationToken cancellationToken = default);
/// <summary>
/// Imports source content from an exported snapshot.
/// </summary>
/// <param name="inputStream">Stream to read content from.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Imported snapshot.</returns>
Task<SourceSnapshot> ImportAsync(
Stream inputStream,
CancellationToken cancellationToken = default);
}
/// <summary>
/// Options for feed snapshot creation.
/// </summary>
public sealed record FeedSnapshotOptions
{
/// <summary>
/// Whether to include full content in the snapshot (vs. just metadata).
/// </summary>
public bool IncludeContent { get; init; } = false;
/// <summary>
/// Whether to compress exported bundles.
/// </summary>
public bool CompressExport { get; init; } = true;
/// <summary>
/// Compression algorithm for exports.
/// </summary>
public CompressionAlgorithm Compression { get; init; } = CompressionAlgorithm.Zstd;
/// <summary>
/// Maximum age of snapshot before it's considered stale.
/// </summary>
public TimeSpan? MaxSnapshotAge { get; init; }
/// <summary>
/// Whether to verify snapshot integrity on import.
/// </summary>
public bool VerifyOnImport { get; init; } = true;
}

429
src/__Libraries/StellaOps.Replay.Core/Validation/DeterminismManifestValidator.cs Normal file
View File

@@ -0,0 +1,429 @@
// -----------------------------------------------------------------------------
// DeterminismManifestValidator.cs
// Sprint: SPRINT_20251226_007_BE_determinism_gaps
// Task: DET-GAP-10
// Description: Validator for determinism manifest compliance
// -----------------------------------------------------------------------------
using System.Collections.Immutable;
using System.Text.Json;
using System.Text.RegularExpressions;
namespace StellaOps.Replay.Core.Validation;
/// <summary>
/// Validates determinism manifests against the formal schema.
/// </summary>
public sealed partial class DeterminismManifestValidator
{
private const string SchemaVersion = "1.0";
private static readonly ImmutableHashSet<string> ValidArtifactTypes = ImmutableHashSet.Create(
StringComparer.OrdinalIgnoreCase,
"sbom", "vex", "csaf", "verdict", "evidence-bundle",
"airgap-bundle", "advisory-normalized", "attestation", "other");
private static readonly ImmutableHashSet<string> ValidHashAlgorithms = ImmutableHashSet.Create(
StringComparer.Ordinal,
"SHA-256", "SHA-384", "SHA-512");
private static readonly ImmutableHashSet<string> ValidEncodings = ImmutableHashSet.Create(
StringComparer.Ordinal,
"hex", "base64");
private static readonly ImmutableHashSet<string> ValidOrderingGuarantees = ImmutableHashSet.Create(
StringComparer.Ordinal,
"stable", "sorted", "insertion", "unspecified");
[GeneratedRegex(@"^[0-9a-f]{64,128}$", RegexOptions.IgnoreCase | RegexOptions.Compiled)]
private static partial Regex HexHashPattern();
[GeneratedRegex(@"^[0-9a-f]{40,64}$", RegexOptions.IgnoreCase | RegexOptions.Compiled)]
private static partial Regex GitShaPattern();
[GeneratedRegex(@"^sha256:[0-9a-f]{64}$", RegexOptions.IgnoreCase | RegexOptions.Compiled)]
private static partial Regex Sha256DigestPattern();
/// <summary>
/// Validates a determinism manifest JSON document.
/// </summary>
public ValidationResult Validate(JsonDocument document)
{
ArgumentNullException.ThrowIfNull(document);
var errors = new List<ValidationError>();
var warnings = new List<ValidationWarning>();
var root = document.RootElement;
// Required fields
ValidateRequired(root, "schemaVersion", errors);
ValidateRequired(root, "artifact", errors);
ValidateRequired(root, "canonicalHash", errors);
ValidateRequired(root, "toolchain", errors);
ValidateRequired(root, "generatedAt", errors);
// Schema version
if (root.TryGetProperty("schemaVersion", out var schemaVersion))
{
if (schemaVersion.GetString() != SchemaVersion)
{
errors.Add(new ValidationError(
"schemaVersion",
$"Unsupported schema version: expected '{SchemaVersion}', got '{schemaVersion.GetString()}'"));
}
}
// Artifact validation
if (root.TryGetProperty("artifact", out var artifact))
{
ValidateArtifact(artifact, errors, warnings);
}
// Canonical hash validation
if (root.TryGetProperty("canonicalHash", out var canonicalHash))
{
ValidateCanonicalHash(canonicalHash, errors);
}
// Toolchain validation
if (root.TryGetProperty("toolchain", out var toolchain))
{
ValidateToolchain(toolchain, errors, warnings);
}
// Generated at validation
if (root.TryGetProperty("generatedAt", out var generatedAt))
{
if (!DateTimeOffset.TryParse(generatedAt.GetString(), out _))
{
errors.Add(new ValidationError(
"generatedAt",
"Invalid ISO 8601 timestamp format"));
}
}
// Inputs validation (optional)
if (root.TryGetProperty("inputs", out var inputs))
{
ValidateInputs(inputs, errors, warnings);
}
// Reproducibility validation (optional)
if (root.TryGetProperty("reproducibility", out var reproducibility))
{
ValidateReproducibility(reproducibility, errors, warnings);
}
// Verification validation (optional)
if (root.TryGetProperty("verification", out var verification))
{
ValidateVerification(verification, warnings);
}
return new ValidationResult
{
IsValid = errors.Count == 0,
Errors = errors.ToImmutableArray(),
Warnings = warnings.ToImmutableArray()
};
}
/// <summary>
/// Validates a determinism manifest from JSON string.
/// </summary>
public ValidationResult Validate(string json)
{
ArgumentException.ThrowIfNullOrWhiteSpace(json);
try
{
using var document = JsonDocument.Parse(json);
return Validate(document);
}
catch (JsonException ex)
{
return new ValidationResult
{
IsValid = false,
Errors = [new ValidationError("$", $"Invalid JSON: {ex.Message}")],
Warnings = []
};
}
}
/// <summary>
/// Validates a determinism manifest from UTF-8 bytes.
/// </summary>
public ValidationResult Validate(ReadOnlySpan<byte> utf8Json)
{
try
{
// Convert to Memory<byte> for JsonDocument.Parse compatibility
using var document = JsonDocument.Parse(utf8Json.ToArray());
return Validate(document);
}
catch (JsonException ex)
{
return new ValidationResult
{
IsValid = false,
Errors = [new ValidationError("$", $"Invalid JSON: {ex.Message}")],
Warnings = []
};
}
}
private static void ValidateRequired(JsonElement element, string propertyName, List<ValidationError> errors)
{
if (!element.TryGetProperty(propertyName, out _))
{
errors.Add(new ValidationError(propertyName, $"Required property '{propertyName}' is missing"));
}
}
private static void ValidateArtifact(JsonElement artifact, List<ValidationError> errors, List<ValidationWarning> warnings)
{
// Required artifact fields
ValidateRequired(artifact, "type", errors);
ValidateRequired(artifact, "name", errors);
ValidateRequired(artifact, "version", errors);
// Artifact type validation
if (artifact.TryGetProperty("type", out var type))
{
var typeValue = type.GetString();
if (string.IsNullOrWhiteSpace(typeValue) || !ValidArtifactTypes.Contains(typeValue))
{
errors.Add(new ValidationError(
"artifact.type",
$"Invalid artifact type: '{typeValue}'. Valid types: {string.Join(", ", ValidArtifactTypes)}"));
}
}
// Name must be non-empty
if (artifact.TryGetProperty("name", out var name))
{
if (string.IsNullOrWhiteSpace(name.GetString()))
{
errors.Add(new ValidationError("artifact.name", "Artifact name cannot be empty"));
}
}
// Recommend format for certain artifact types
if (artifact.TryGetProperty("type", out var artifactType))
{
var typeStr = artifactType.GetString();
if ((typeStr == "sbom" || typeStr == "vex") && !artifact.TryGetProperty("format", out _))
{
warnings.Add(new ValidationWarning(
"artifact.format",
$"Recommend specifying format for {typeStr} artifacts (e.g., 'SPDX 3.0.1', 'CycloneDX 1.6', 'OpenVEX')"));
}
}
}
private static void ValidateCanonicalHash(JsonElement canonicalHash, List<ValidationError> errors)
{
ValidateRequired(canonicalHash, "algorithm", errors);
ValidateRequired(canonicalHash, "value", errors);
ValidateRequired(canonicalHash, "encoding", errors);
// Algorithm validation
if (canonicalHash.TryGetProperty("algorithm", out var algorithm))
{
var algValue = algorithm.GetString();
if (!ValidHashAlgorithms.Contains(algValue ?? string.Empty))
{
errors.Add(new ValidationError(
"canonicalHash.algorithm",
$"Invalid hash algorithm: '{algValue}'. Valid algorithms: {string.Join(", ", ValidHashAlgorithms)}"));
}
}
// Encoding validation
if (canonicalHash.TryGetProperty("encoding", out var encoding))
{
var encValue = encoding.GetString();
if (!ValidEncodings.Contains(encValue ?? string.Empty))
{
errors.Add(new ValidationError(
"canonicalHash.encoding",
$"Invalid encoding: '{encValue}'. Valid encodings: {string.Join(", ", ValidEncodings)}"));
}
}
// Value format validation
if (canonicalHash.TryGetProperty("value", out var value) &&
canonicalHash.TryGetProperty("encoding", out var enc))
{
var valueStr = value.GetString() ?? string.Empty;
var encStr = enc.GetString();
if (encStr == "hex" && !HexHashPattern().IsMatch(valueStr))
{
errors.Add(new ValidationError(
"canonicalHash.value",
"Hash value does not match expected hex pattern (64-128 hex characters)"));
}
}
}
private static void ValidateToolchain(JsonElement toolchain, List<ValidationError> errors, List<ValidationWarning> warnings)
{
ValidateRequired(toolchain, "platform", errors);
ValidateRequired(toolchain, "components", errors);
// Components should be an array
if (toolchain.TryGetProperty("components", out var components))
{
if (components.ValueKind != JsonValueKind.Array)
{
errors.Add(new ValidationError(
"toolchain.components",
"Components must be an array"));
}
else if (components.GetArrayLength() == 0)
{
warnings.Add(new ValidationWarning(
"toolchain.components",
"Components array is empty - consider adding tool versions for reproducibility"));
}
else
{
var index = 0;
foreach (var component in components.EnumerateArray())
{
ValidateRequired(component, "name", errors);
ValidateRequired(component, "version", errors);
index++;
}
}
}
}
private static void ValidateInputs(JsonElement inputs, List<ValidationError> errors, List<ValidationWarning> warnings)
{
// feedSnapshotHash validation
if (inputs.TryGetProperty("feedSnapshotHash", out var feedHash))
{
var hashStr = feedHash.GetString() ?? string.Empty;
if (!HexHashPattern().IsMatch(hashStr))
{
errors.Add(new ValidationError(
"inputs.feedSnapshotHash",
"Feed snapshot hash must be 64 hex characters"));
}
}
// policyManifestHash validation
if (inputs.TryGetProperty("policyManifestHash", out var policyHash))
{
var hashStr = policyHash.GetString() ?? string.Empty;
if (!HexHashPattern().IsMatch(hashStr))
{
errors.Add(new ValidationError(
"inputs.policyManifestHash",
"Policy manifest hash must be 64 hex characters"));
}
}
// sourceCodeHash validation
if (inputs.TryGetProperty("sourceCodeHash", out var sourceHash))
{
var hashStr = sourceHash.GetString() ?? string.Empty;
if (!GitShaPattern().IsMatch(hashStr))
{
errors.Add(new ValidationError(
"inputs.sourceCodeHash",
"Source code hash must be 40-64 hex characters (git SHA format)"));
}
}
// baseImageDigest validation
if (inputs.TryGetProperty("baseImageDigest", out var baseImage))
{
var digestStr = baseImage.GetString() ?? string.Empty;
if (!Sha256DigestPattern().IsMatch(digestStr))
{
errors.Add(new ValidationError(
"inputs.baseImageDigest",
"Base image digest must be in format 'sha256:64hexchars'"));
}
}
// Warn if no inputs specified
var hasAnyInput = inputs.EnumerateObject().Any();
if (!hasAnyInput)
{
warnings.Add(new ValidationWarning(
"inputs",
"No inputs specified - consider adding feed/policy/source hashes for full reproducibility"));
}
}
private static void ValidateReproducibility(JsonElement reproducibility, List<ValidationError> errors, List<ValidationWarning> warnings)
{
// orderingGuarantee validation
if (reproducibility.TryGetProperty("orderingGuarantee", out var ordering))
{
var orderStr = ordering.GetString();
if (!ValidOrderingGuarantees.Contains(orderStr ?? string.Empty))
{
errors.Add(new ValidationError(
"reproducibility.orderingGuarantee",
$"Invalid ordering guarantee: '{orderStr}'. Valid values: {string.Join(", ", ValidOrderingGuarantees)}"));
}
}
// Warn if not using stable ordering
if (!reproducibility.TryGetProperty("orderingGuarantee", out _))
{
warnings.Add(new ValidationWarning(
"reproducibility.orderingGuarantee",
"Consider specifying orderingGuarantee for deterministic output"));
}
// normalizationRules should be an array
if (reproducibility.TryGetProperty("normalizationRules", out var rules))
{
if (rules.ValueKind != JsonValueKind.Array)
{
errors.Add(new ValidationError(
"reproducibility.normalizationRules",
"Normalization rules must be an array"));
}
}
}
private static void ValidateVerification(JsonElement verification, List<ValidationWarning> warnings)
{
// Warn if command is specified but expectedHash is missing
if (verification.TryGetProperty("command", out _) &&
!verification.TryGetProperty("expectedHash", out _))
{
warnings.Add(new ValidationWarning(
"verification.expectedHash",
"Command specified without expectedHash - consider adding for verification"));
}
}
}
/// <summary>
/// Result of manifest validation.
/// </summary>
public sealed record ValidationResult
{
public required bool IsValid { get; init; }
public required ImmutableArray<ValidationError> Errors { get; init; }
public required ImmutableArray<ValidationWarning> Warnings { get; init; }
}
/// <summary>
/// Validation error.
/// </summary>
public sealed record ValidationError(string Path, string Message);
/// <summary>
/// Validation warning.
/// </summary>
public sealed record ValidationWarning(string Path, string Message);