consolidation of some of the modules, localization fixes, product advisories work, qa work

This commit is contained in:
master
2026-03-05 03:54:22 +02:00
parent 7bafcc3eef
commit 8e1cb9448d
3878 changed files with 72600 additions and 46861 deletions

View File

@@ -58,8 +58,9 @@ builder.Services.AddSingleton<IReplayExecutor, ReplayExecutor>();
builder.Services.AddSingleton<IAuditBundleReader, AuditBundleReader>();
builder.Services.AddSingleton<IVerdictReplayPredicate, VerdictReplayPredicate>();
builder.Services.AddSingleton<IFeedSnapshotBlobStore, InMemoryFeedSnapshotBlobStore>();
builder.Services.AddSingleton<IFeedSnapshotIndexStore, InMemoryFeedSnapshotIndexStore>();
var replayStorageDriver = ResolveStorageDriver(builder.Configuration, "Replay");
RegisterSnapshotIndexStore(builder.Services, builder.Configuration, builder.Environment.IsDevelopment(), replayStorageDriver);
RegisterSnapshotBlobStore(builder.Services, builder.Configuration, "Replay");
builder.Services.AddSingleton(new FeedSnapshotServiceOptions());
builder.Services.AddSingleton<FeedSnapshotService>();
builder.Services.AddSingleton<IAdvisoryExtractor, JsonAdvisoryExtractor>();
@@ -435,6 +436,106 @@ await app.LoadTranslationsAsync();
app.TryRefreshStellaRouterEndpoints(routerEnabled);
await app.RunAsync().ConfigureAwait(false);
static void RegisterSnapshotIndexStore(IServiceCollection services, IConfiguration configuration, bool isDevelopment, string storageDriver)
{
if (string.Equals(storageDriver, "postgres", StringComparison.OrdinalIgnoreCase))
{
var connectionString = ResolvePostgresConnectionString(configuration, "Replay");
if (string.IsNullOrWhiteSpace(connectionString))
{
if (!isDevelopment)
{
throw new InvalidOperationException(
"Replay requires PostgreSQL connection settings in non-development mode. " +
"Set ConnectionStrings:Default or Replay:Storage:Postgres:ConnectionString.");
}
services.AddSingleton<IFeedSnapshotIndexStore, InMemoryFeedSnapshotIndexStore>();
return;
}
services.AddSingleton<IFeedSnapshotIndexStore>(_ => new PostgresFeedSnapshotIndexStore(connectionString));
return;
}
if (string.Equals(storageDriver, "inmemory", StringComparison.OrdinalIgnoreCase))
{
services.AddSingleton<IFeedSnapshotIndexStore, InMemoryFeedSnapshotIndexStore>();
return;
}
throw new InvalidOperationException(
$"Unsupported Replay storage driver '{storageDriver}'. Allowed values: postgres, inmemory.");
}
static void RegisterSnapshotBlobStore(IServiceCollection services, IConfiguration configuration, string serviceName)
{
var objectStoreDriver = ResolveObjectStoreDriver(configuration, serviceName);
if (string.Equals(objectStoreDriver, "seed-fs", StringComparison.OrdinalIgnoreCase))
{
var rootPath = ResolveSeedFsRootPath(configuration, serviceName)
?? Path.Combine("data", "replay", "snapshots");
services.AddSingleton<IFeedSnapshotBlobStore>(_ => new SeedFsFeedSnapshotBlobStore(rootPath));
return;
}
if (string.Equals(objectStoreDriver, "rustfs", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException(
"Replay RustFS blob driver is not supported in the current runtime contract. Use seed-fs.");
}
throw new InvalidOperationException(
$"Unsupported Replay object store driver '{objectStoreDriver}'. Allowed values: seed-fs.");
}
static string ResolveStorageDriver(IConfiguration configuration, string serviceName)
{
return FirstNonEmpty(
configuration["Storage:Driver"],
configuration[$"{serviceName}:Storage:Driver"])
?? "postgres";
}
static string ResolveObjectStoreDriver(IConfiguration configuration, string serviceName)
{
return FirstNonEmpty(
configuration[$"{serviceName}:Storage:ObjectStore:Driver"],
configuration["Storage:ObjectStore:Driver"])
?? "seed-fs";
}
static string? ResolveSeedFsRootPath(IConfiguration configuration, string serviceName)
{
return FirstNonEmpty(
configuration[$"{serviceName}:Storage:ObjectStore:SeedFs:RootPath"],
configuration["Storage:ObjectStore:SeedFs:RootPath"],
configuration[$"{serviceName}:Snapshots:SeedFs:RootPath"]);
}
static string? ResolvePostgresConnectionString(IConfiguration configuration, string serviceName)
{
return FirstNonEmpty(
configuration[$"{serviceName}:Storage:Postgres:ConnectionString"],
configuration["Storage:Postgres:ConnectionString"],
configuration[$"Postgres:{serviceName}:ConnectionString"],
configuration[$"ConnectionStrings:{serviceName}"],
configuration["ConnectionStrings:Default"]);
}
static string? FirstNonEmpty(params string?[] values)
{
foreach (var value in values)
{
if (!string.IsNullOrWhiteSpace(value))
{
return value;
}
}
return null;
}
static bool TryGetTenant(HttpContext httpContext, out ProblemHttpResult? problem, out string tenantId)
{
tenantId = string.Empty;

View File

@@ -0,0 +1,311 @@
using Npgsql;
using StellaOps.Replay.Core.FeedSnapshots;
using System.Collections.Immutable;
using System.Text.Json;
namespace StellaOps.Replay.WebService;
public sealed class PostgresFeedSnapshotIndexStore : IFeedSnapshotIndexStore, IAsyncDisposable
{
private readonly NpgsqlDataSource _dataSource;
private readonly object _initGate = new();
private bool _tableInitialized;
public PostgresFeedSnapshotIndexStore(string connectionString)
{
ArgumentException.ThrowIfNullOrWhiteSpace(connectionString);
_dataSource = NpgsqlDataSource.Create(connectionString);
}
public async Task IndexSnapshotAsync(FeedSnapshotIndexEntry entry, CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(entry);
await EnsureTableAsync(ct).ConfigureAwait(false);
const string sql = """
INSERT INTO replay.feed_snapshot_index (
provider_id,
digest,
captured_at,
epoch_timestamp
) VALUES (
@provider_id,
@digest,
@captured_at,
@epoch_timestamp
)
ON CONFLICT (provider_id, captured_at, digest) DO NOTHING;
""";
await using var connection = await _dataSource.OpenConnectionAsync(ct).ConfigureAwait(false);
await using var command = new NpgsqlCommand(sql, connection);
command.Parameters.AddWithValue("provider_id", entry.ProviderId);
command.Parameters.AddWithValue("digest", entry.Digest);
command.Parameters.AddWithValue("captured_at", entry.CapturedAt);
command.Parameters.AddWithValue("epoch_timestamp", entry.EpochTimestamp);
await command.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
}
public async Task<FeedSnapshotIndexEntry?> FindSnapshotAtTimeAsync(
string providerId,
DateTimeOffset pointInTime,
CancellationToken ct = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(providerId);
await EnsureTableAsync(ct).ConfigureAwait(false);
const string sql = """
SELECT provider_id, digest, captured_at, epoch_timestamp
FROM replay.feed_snapshot_index
WHERE provider_id = @provider_id
AND captured_at <= @point_in_time
ORDER BY captured_at DESC, digest ASC
LIMIT 1;
""";
await using var connection = await _dataSource.OpenConnectionAsync(ct).ConfigureAwait(false);
await using var command = new NpgsqlCommand(sql, connection);
command.Parameters.AddWithValue("provider_id", providerId);
command.Parameters.AddWithValue("point_in_time", pointInTime);
await using var reader = await command.ExecuteReaderAsync(ct).ConfigureAwait(false);
if (!await reader.ReadAsync(ct).ConfigureAwait(false))
{
return null;
}
return new FeedSnapshotIndexEntry
{
ProviderId = reader.GetString(0),
Digest = reader.GetString(1),
CapturedAt = reader.GetFieldValue<DateTimeOffset>(2),
EpochTimestamp = reader.GetFieldValue<DateTimeOffset>(3),
};
}
public async Task<ImmutableArray<FeedSnapshotIndexEntry>> ListSnapshotsAsync(
string providerId,
DateTimeOffset from,
DateTimeOffset to,
int limit,
CancellationToken ct = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(providerId);
await EnsureTableAsync(ct).ConfigureAwait(false);
const string sql = """
SELECT provider_id, digest, captured_at, epoch_timestamp
FROM replay.feed_snapshot_index
WHERE provider_id = @provider_id
AND captured_at >= @from
AND captured_at <= @to
ORDER BY captured_at ASC, digest ASC
LIMIT @limit;
""";
var effectiveLimit = limit <= 0 ? 1000 : limit;
var results = ImmutableArray.CreateBuilder<FeedSnapshotIndexEntry>(effectiveLimit);
await using var connection = await _dataSource.OpenConnectionAsync(ct).ConfigureAwait(false);
await using var command = new NpgsqlCommand(sql, connection);
command.Parameters.AddWithValue("provider_id", providerId);
command.Parameters.AddWithValue("from", from);
command.Parameters.AddWithValue("to", to);
command.Parameters.AddWithValue("limit", effectiveLimit);
await using var reader = await command.ExecuteReaderAsync(ct).ConfigureAwait(false);
while (await reader.ReadAsync(ct).ConfigureAwait(false))
{
results.Add(new FeedSnapshotIndexEntry
{
ProviderId = reader.GetString(0),
Digest = reader.GetString(1),
CapturedAt = reader.GetFieldValue<DateTimeOffset>(2),
EpochTimestamp = reader.GetFieldValue<DateTimeOffset>(3),
});
}
return results.ToImmutable();
}
public ValueTask DisposeAsync()
{
return _dataSource.DisposeAsync();
}
private async Task EnsureTableAsync(CancellationToken ct)
{
lock (_initGate)
{
if (_tableInitialized)
{
return;
}
}
const string ddl = """
CREATE SCHEMA IF NOT EXISTS replay;
CREATE TABLE IF NOT EXISTS replay.feed_snapshot_index (
provider_id TEXT NOT NULL,
digest TEXT NOT NULL,
captured_at TIMESTAMPTZ NOT NULL,
epoch_timestamp TIMESTAMPTZ NOT NULL,
PRIMARY KEY (provider_id, captured_at, digest)
);
CREATE INDEX IF NOT EXISTS idx_replay_snapshot_index_lookup
ON replay.feed_snapshot_index (provider_id, captured_at DESC, digest ASC);
""";
await using var connection = await _dataSource.OpenConnectionAsync(ct).ConfigureAwait(false);
await using var command = new NpgsqlCommand(ddl, connection);
await command.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
lock (_initGate)
{
_tableInitialized = true;
}
}
}
public sealed class SeedFsFeedSnapshotBlobStore : IFeedSnapshotBlobStore
{
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
{
WriteIndented = false,
};
private readonly string _rootPath;
public SeedFsFeedSnapshotBlobStore(string rootPath)
{
ArgumentException.ThrowIfNullOrWhiteSpace(rootPath);
_rootPath = Path.GetFullPath(rootPath);
Directory.CreateDirectory(_rootPath);
}
public async Task StoreAsync(FeedSnapshotBlob blob, CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(blob);
var key = ToSafeKey(blob.Digest);
var contentPath = GetContentPath(key);
var metadataPath = GetMetadataPath(key);
Directory.CreateDirectory(Path.GetDirectoryName(contentPath)!);
await File.WriteAllBytesAsync(contentPath, blob.Content, ct).ConfigureAwait(false);
var metadata = FeedSnapshotBlobMetadata.FromBlob(blob);
var json = JsonSerializer.Serialize(metadata, JsonOptions);
await File.WriteAllTextAsync(metadataPath, json, ct).ConfigureAwait(false);
}
public async Task<FeedSnapshotBlob?> GetByDigestAsync(string digest, CancellationToken ct = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(digest);
var key = ToSafeKey(digest);
var contentPath = GetContentPath(key);
var metadataPath = GetMetadataPath(key);
if (!File.Exists(contentPath) || !File.Exists(metadataPath))
{
return null;
}
var content = await File.ReadAllBytesAsync(contentPath, ct).ConfigureAwait(false);
var json = await File.ReadAllTextAsync(metadataPath, ct).ConfigureAwait(false);
var metadata = JsonSerializer.Deserialize<FeedSnapshotBlobMetadata>(json, JsonOptions);
if (metadata is null)
{
return null;
}
return metadata.ToBlob(content);
}
public Task<bool> ExistsAsync(string digest, CancellationToken ct = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(digest);
var key = ToSafeKey(digest);
var exists = File.Exists(GetContentPath(key)) && File.Exists(GetMetadataPath(key));
return Task.FromResult(exists);
}
public Task DeleteAsync(string digest, CancellationToken ct = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(digest);
var key = ToSafeKey(digest);
var contentPath = GetContentPath(key);
var metadataPath = GetMetadataPath(key);
if (File.Exists(contentPath))
{
File.Delete(contentPath);
}
if (File.Exists(metadataPath))
{
File.Delete(metadataPath);
}
return Task.CompletedTask;
}
private string GetContentPath(string key) => Path.Combine(_rootPath, "snapshots", $"{key}.bin");
private string GetMetadataPath(string key) => Path.Combine(_rootPath, "snapshots", $"{key}.meta.json");
private static string ToSafeKey(string digest)
{
var value = digest.Trim();
value = value.Replace(':', '_').Replace('/', '_');
foreach (var invalid in Path.GetInvalidFileNameChars())
{
value = value.Replace(invalid, '_');
}
return string.IsNullOrWhiteSpace(value) ? "snapshot" : value;
}
private sealed record FeedSnapshotBlobMetadata(
string Digest,
string ProviderId,
string? ProviderName,
string? FeedType,
DateTimeOffset CapturedAt,
DateTimeOffset EpochTimestamp,
Dictionary<string, string> Metadata,
string ContentHash,
FeedSnapshotFormat Format)
{
public static FeedSnapshotBlobMetadata FromBlob(FeedSnapshotBlob blob)
{
return new FeedSnapshotBlobMetadata(
blob.Digest,
blob.ProviderId,
blob.ProviderName,
blob.FeedType,
blob.CapturedAt,
blob.EpochTimestamp,
blob.Metadata.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal),
blob.ContentHash,
blob.Format);
}
public FeedSnapshotBlob ToBlob(byte[] content)
{
return new FeedSnapshotBlob
{
Digest = Digest,
ProviderId = ProviderId,
ProviderName = ProviderName,
FeedType = FeedType,
Content = content,
CapturedAt = CapturedAt,
EpochTimestamp = EpochTimestamp,
Metadata = Metadata.ToImmutableDictionary(StringComparer.Ordinal),
ContentHash = ContentHash,
Format = Format
};
}
}
}

View File

@@ -8,6 +8,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Npgsql" />
<PackageReference Include="Serilog.AspNetCore" />
<PackageReference Include="Serilog.Sinks.Console" />
</ItemGroup>

View File

@@ -6,3 +6,4 @@ Source of truth: `docs/implplan/SPRINT_20260130_002_Tools_csproj_remediation_sol
| --- | --- | --- |
| REMED-05 | TODO | Remediation checklist: docs/implplan/audits/csproj-standards/remediation/checklists/src/Replay/StellaOps.Replay.WebService/StellaOps.Replay.WebService.md. |
| REMED-06 | DONE | SOLID review notes captured for SPRINT_20260130_002. |
| SPRINT-312-006 | DONE | Added Postgres snapshot index + seed-fs snapshot blob stores and wired storage-driver registration in webservice startup. |

View File

@@ -0,0 +1,160 @@
using System.Collections.Immutable;
using FluentAssertions;
using StellaOps.Infrastructure.Postgres.Testing;
using StellaOps.Replay.Core.FeedSnapshots;
using StellaOps.Replay.WebService;
using StellaOps.TestKit;
using Xunit;
namespace StellaOps.Replay.Core.Tests.FeedSnapshots;
[Collection(ReplayPostgresCollection.Name)]
public sealed class PostgresFeedSnapshotIndexStoreTests : IAsyncLifetime
{
private readonly ReplayPostgresFixture _fixture;
private readonly PostgresFeedSnapshotIndexStore _store;
public PostgresFeedSnapshotIndexStoreTests(ReplayPostgresFixture fixture)
{
_fixture = fixture;
_store = new PostgresFeedSnapshotIndexStore(_fixture.ConnectionString);
}
public async ValueTask InitializeAsync()
{
await _fixture.ExecuteSqlAsync("DROP SCHEMA IF EXISTS replay CASCADE;");
}
public async ValueTask DisposeAsync()
{
await _store.DisposeAsync();
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task FindSnapshotAtTimeAsync_ReturnsLatestSnapshotAtOrBeforePoint()
{
var baseTime = new DateTimeOffset(2026, 03, 05, 09, 00, 00, TimeSpan.Zero);
await _store.IndexSnapshotAsync(new FeedSnapshotIndexEntry
{
ProviderId = "nvd",
Digest = "sha256:a",
CapturedAt = baseTime,
EpochTimestamp = baseTime,
});
await _store.IndexSnapshotAsync(new FeedSnapshotIndexEntry
{
ProviderId = "nvd",
Digest = "sha256:b",
CapturedAt = baseTime.AddMinutes(10),
EpochTimestamp = baseTime.AddMinutes(10),
});
var atFive = await _store.FindSnapshotAtTimeAsync("nvd", baseTime.AddMinutes(5));
var atTen = await _store.FindSnapshotAtTimeAsync("nvd", baseTime.AddMinutes(10));
var atFifteen = await _store.FindSnapshotAtTimeAsync("nvd", baseTime.AddMinutes(15));
atFive.Should().NotBeNull();
atFive!.Digest.Should().Be("sha256:a");
atTen.Should().NotBeNull();
atTen!.Digest.Should().Be("sha256:b");
atFifteen.Should().NotBeNull();
atFifteen!.Digest.Should().Be("sha256:b");
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ListSnapshotsAsync_ReturnsOrderedRowsWithinTimeRange()
{
var baseTime = new DateTimeOffset(2026, 03, 05, 11, 00, 00, TimeSpan.Zero);
await _store.IndexSnapshotAsync(new FeedSnapshotIndexEntry
{
ProviderId = "ghsa",
Digest = "sha256:1",
CapturedAt = baseTime,
EpochTimestamp = baseTime,
});
await _store.IndexSnapshotAsync(new FeedSnapshotIndexEntry
{
ProviderId = "ghsa",
Digest = "sha256:2",
CapturedAt = baseTime.AddMinutes(5),
EpochTimestamp = baseTime.AddMinutes(5),
});
await _store.IndexSnapshotAsync(new FeedSnapshotIndexEntry
{
ProviderId = "ghsa",
Digest = "sha256:3",
CapturedAt = baseTime.AddMinutes(10),
EpochTimestamp = baseTime.AddMinutes(10),
});
var listed = await _store.ListSnapshotsAsync(
"ghsa",
from: baseTime.AddMinutes(1),
to: baseTime.AddMinutes(10),
limit: 5);
listed.Select(x => x.Digest).Should().Equal("sha256:2", "sha256:3");
}
}
public sealed class SeedFsFeedSnapshotBlobStoreTests
{
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task StoreGetExistsDelete_RoundTripsBlobContentAndMetadata()
{
var rootPath = Path.Combine(Path.GetTempPath(), "stellaops-replay-seedfs", Guid.NewGuid().ToString("N"));
var store = new SeedFsFeedSnapshotBlobStore(rootPath);
var capturedAt = new DateTimeOffset(2026, 03, 05, 12, 00, 00, TimeSpan.Zero);
var blob = new FeedSnapshotBlob
{
Digest = "sha256:test-digest",
ProviderId = "nvd",
ProviderName = "NVD",
FeedType = "advisory",
Content = new byte[] { 1, 3, 5, 7 },
CapturedAt = capturedAt,
EpochTimestamp = capturedAt,
Metadata = new Dictionary<string, string>(StringComparer.Ordinal)
.ToImmutableDictionary(StringComparer.Ordinal),
ContentHash = "sha256:test-hash",
Format = FeedSnapshotFormat.CanonicalJson,
};
await store.StoreAsync(blob);
var exists = await store.ExistsAsync(blob.Digest);
var loaded = await store.GetByDigestAsync(blob.Digest);
exists.Should().BeTrue();
loaded.Should().NotBeNull();
loaded!.Digest.Should().Be(blob.Digest);
loaded.ProviderId.Should().Be(blob.ProviderId);
loaded.Content.Should().Equal(blob.Content);
loaded.ContentHash.Should().Be(blob.ContentHash);
await store.DeleteAsync(blob.Digest);
var existsAfterDelete = await store.ExistsAsync(blob.Digest);
existsAfterDelete.Should().BeFalse();
if (Directory.Exists(rootPath))
{
Directory.Delete(rootPath, recursive: true);
}
}
}
public sealed class ReplayPostgresFixture : PostgresIntegrationFixture, ICollectionFixture<ReplayPostgresFixture>
{
protected override System.Reflection.Assembly? GetMigrationAssembly() => null;
protected override string GetModuleName() => "Replay";
}
[CollectionDefinition(Name)]
public sealed class ReplayPostgresCollection : ICollectionFixture<ReplayPostgresFixture>
{
public const string Name = "ReplayPostgres";
}

View File

@@ -18,6 +18,7 @@
<ItemGroup>
<ProjectReference Include="../../__Libraries/StellaOps.Replay.Core/StellaOps.Replay.Core.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.AuditPack/StellaOps.AuditPack.csproj" />
<ProjectReference Include="../../../__Tests/__Libraries/StellaOps.Infrastructure.Postgres.Testing/StellaOps.Infrastructure.Postgres.Testing.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.TestKit/StellaOps.TestKit.csproj" />
<ProjectReference Include="../../StellaOps.Replay.WebService/StellaOps.Replay.WebService.csproj" />
</ItemGroup>

View File

@@ -6,3 +6,4 @@ Source of truth: `docs/implplan/SPRINT_20260130_002_Tools_csproj_remediation_sol
| --- | --- | --- |
| REMED-05 | TODO | Remediation checklist: docs/implplan/audits/csproj-standards/remediation/checklists/src/Replay/__Tests/StellaOps.Replay.Core.Tests/StellaOps.Replay.Core.Tests.md. |
| REMED-06 | DONE | SOLID review notes captured for SPRINT_20260130_002. |
| SPRINT-312-006 | DONE | Added `ReplayFeedSnapshotStoresTests` and validated Postgres index + seed-fs blob stores via class-targeted xUnit execution (3/3 pass). |