consolidation of some of the modules, localization fixes, product advisories work, qa work

2026-03-05 03:54:22 +02:00
parent 7bafcc3eef
commit 8e1cb9448d
3878 changed files with 72600 additions and 46861 deletions
--- a/src/Replay/StellaOps.Replay.WebService/Program.cs
+++ b/src/Replay/StellaOps.Replay.WebService/Program.cs
@@ -58,8 +58,9 @@ builder.Services.AddSingleton<IReplayExecutor, ReplayExecutor>();
 builder.Services.AddSingleton<IAuditBundleReader, AuditBundleReader>();
 builder.Services.AddSingleton<IVerdictReplayPredicate, VerdictReplayPredicate>();

-builder.Services.AddSingleton<IFeedSnapshotBlobStore, InMemoryFeedSnapshotBlobStore>();
-builder.Services.AddSingleton<IFeedSnapshotIndexStore, InMemoryFeedSnapshotIndexStore>();
+var replayStorageDriver = ResolveStorageDriver(builder.Configuration, "Replay");
+RegisterSnapshotIndexStore(builder.Services, builder.Configuration, builder.Environment.IsDevelopment(), replayStorageDriver);
+RegisterSnapshotBlobStore(builder.Services, builder.Configuration, "Replay");
 builder.Services.AddSingleton(new FeedSnapshotServiceOptions());
 builder.Services.AddSingleton<FeedSnapshotService>();
 builder.Services.AddSingleton<IAdvisoryExtractor, JsonAdvisoryExtractor>();
@@ -435,6 +436,106 @@ await app.LoadTranslationsAsync();
 app.TryRefreshStellaRouterEndpoints(routerEnabled);
 await app.RunAsync().ConfigureAwait(false);

+static void RegisterSnapshotIndexStore(IServiceCollection services, IConfiguration configuration, bool isDevelopment, string storageDriver)
+{
+    if (string.Equals(storageDriver, "postgres", StringComparison.OrdinalIgnoreCase))
+    {
+        var connectionString = ResolvePostgresConnectionString(configuration, "Replay");
+        if (string.IsNullOrWhiteSpace(connectionString))
+        {
+            if (!isDevelopment)
+            {
+                throw new InvalidOperationException(
+                    "Replay requires PostgreSQL connection settings in non-development mode. " +
+                    "Set ConnectionStrings:Default or Replay:Storage:Postgres:ConnectionString.");
+            }
+
+            services.AddSingleton<IFeedSnapshotIndexStore, InMemoryFeedSnapshotIndexStore>();
+            return;
+        }
+
+        services.AddSingleton<IFeedSnapshotIndexStore>(_ => new PostgresFeedSnapshotIndexStore(connectionString));
+        return;
+    }
+
+    if (string.Equals(storageDriver, "inmemory", StringComparison.OrdinalIgnoreCase))
+    {
+        services.AddSingleton<IFeedSnapshotIndexStore, InMemoryFeedSnapshotIndexStore>();
+        return;
+    }
+
+    throw new InvalidOperationException(
+        $"Unsupported Replay storage driver '{storageDriver}'. Allowed values: postgres, inmemory.");
+}
+
+static void RegisterSnapshotBlobStore(IServiceCollection services, IConfiguration configuration, string serviceName)
+{
+    var objectStoreDriver = ResolveObjectStoreDriver(configuration, serviceName);
+    if (string.Equals(objectStoreDriver, "seed-fs", StringComparison.OrdinalIgnoreCase))
+    {
+        var rootPath = ResolveSeedFsRootPath(configuration, serviceName)
+            ?? Path.Combine("data", "replay", "snapshots");
+        services.AddSingleton<IFeedSnapshotBlobStore>(_ => new SeedFsFeedSnapshotBlobStore(rootPath));
+        return;
+    }
+
+    if (string.Equals(objectStoreDriver, "rustfs", StringComparison.OrdinalIgnoreCase))
+    {
+        throw new InvalidOperationException(
+            "Replay RustFS blob driver is not supported in the current runtime contract. Use seed-fs.");
+    }
+
+    throw new InvalidOperationException(
+        $"Unsupported Replay object store driver '{objectStoreDriver}'. Allowed values: seed-fs.");
+}
+
+static string ResolveStorageDriver(IConfiguration configuration, string serviceName)
+{
+    return FirstNonEmpty(
+            configuration["Storage:Driver"],
+            configuration[$"{serviceName}:Storage:Driver"])
+        ?? "postgres";
+}
+
+static string ResolveObjectStoreDriver(IConfiguration configuration, string serviceName)
+{
+    return FirstNonEmpty(
+            configuration[$"{serviceName}:Storage:ObjectStore:Driver"],
+            configuration["Storage:ObjectStore:Driver"])
+        ?? "seed-fs";
+}
+
+static string? ResolveSeedFsRootPath(IConfiguration configuration, string serviceName)
+{
+    return FirstNonEmpty(
+        configuration[$"{serviceName}:Storage:ObjectStore:SeedFs:RootPath"],
+        configuration["Storage:ObjectStore:SeedFs:RootPath"],
+        configuration[$"{serviceName}:Snapshots:SeedFs:RootPath"]);
+}
+
+static string? ResolvePostgresConnectionString(IConfiguration configuration, string serviceName)
+{
+    return FirstNonEmpty(
+        configuration[$"{serviceName}:Storage:Postgres:ConnectionString"],
+        configuration["Storage:Postgres:ConnectionString"],
+        configuration[$"Postgres:{serviceName}:ConnectionString"],
+        configuration[$"ConnectionStrings:{serviceName}"],
+        configuration["ConnectionStrings:Default"]);
+}
+
+static string? FirstNonEmpty(params string?[] values)
+{
+    foreach (var value in values)
+    {
+        if (!string.IsNullOrWhiteSpace(value))
+        {
+            return value;
+        }
+    }
+
+    return null;
+}
+
 static bool TryGetTenant(HttpContext httpContext, out ProblemHttpResult? problem, out string tenantId)
 {
    tenantId = string.Empty;
--- a/src/Replay/StellaOps.Replay.WebService/ReplayFeedSnapshotStores.cs
+++ b/src/Replay/StellaOps.Replay.WebService/ReplayFeedSnapshotStores.cs
@@ -0,0 +1,311 @@
+using Npgsql;
+using StellaOps.Replay.Core.FeedSnapshots;
+using System.Collections.Immutable;
+using System.Text.Json;
+
+namespace StellaOps.Replay.WebService;
+
+public sealed class PostgresFeedSnapshotIndexStore : IFeedSnapshotIndexStore, IAsyncDisposable
+{
+    private readonly NpgsqlDataSource _dataSource;
+    private readonly object _initGate = new();
+    private bool _tableInitialized;
+
+    public PostgresFeedSnapshotIndexStore(string connectionString)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(connectionString);
+        _dataSource = NpgsqlDataSource.Create(connectionString);
+    }
+
+    public async Task IndexSnapshotAsync(FeedSnapshotIndexEntry entry, CancellationToken ct = default)
+    {
+        ArgumentNullException.ThrowIfNull(entry);
+        await EnsureTableAsync(ct).ConfigureAwait(false);
+
+        const string sql = """
+            INSERT INTO replay.feed_snapshot_index (
+                provider_id,
+                digest,
+                captured_at,
+                epoch_timestamp
+            ) VALUES (
+                @provider_id,
+                @digest,
+                @captured_at,
+                @epoch_timestamp
+            )
+            ON CONFLICT (provider_id, captured_at, digest) DO NOTHING;
+            """;
+
+        await using var connection = await _dataSource.OpenConnectionAsync(ct).ConfigureAwait(false);
+        await using var command = new NpgsqlCommand(sql, connection);
+        command.Parameters.AddWithValue("provider_id", entry.ProviderId);
+        command.Parameters.AddWithValue("digest", entry.Digest);
+        command.Parameters.AddWithValue("captured_at", entry.CapturedAt);
+        command.Parameters.AddWithValue("epoch_timestamp", entry.EpochTimestamp);
+        await command.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
+    }
+
+    public async Task<FeedSnapshotIndexEntry?> FindSnapshotAtTimeAsync(
+        string providerId,
+        DateTimeOffset pointInTime,
+        CancellationToken ct = default)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(providerId);
+        await EnsureTableAsync(ct).ConfigureAwait(false);
+
+        const string sql = """
+            SELECT provider_id, digest, captured_at, epoch_timestamp
+            FROM replay.feed_snapshot_index
+            WHERE provider_id = @provider_id
+              AND captured_at <= @point_in_time
+            ORDER BY captured_at DESC, digest ASC
+            LIMIT 1;
+            """;
+
+        await using var connection = await _dataSource.OpenConnectionAsync(ct).ConfigureAwait(false);
+        await using var command = new NpgsqlCommand(sql, connection);
+        command.Parameters.AddWithValue("provider_id", providerId);
+        command.Parameters.AddWithValue("point_in_time", pointInTime);
+
+        await using var reader = await command.ExecuteReaderAsync(ct).ConfigureAwait(false);
+        if (!await reader.ReadAsync(ct).ConfigureAwait(false))
+        {
+            return null;
+        }
+
+        return new FeedSnapshotIndexEntry
+        {
+            ProviderId = reader.GetString(0),
+            Digest = reader.GetString(1),
+            CapturedAt = reader.GetFieldValue<DateTimeOffset>(2),
+            EpochTimestamp = reader.GetFieldValue<DateTimeOffset>(3),
+        };
+    }
+
+    public async Task<ImmutableArray<FeedSnapshotIndexEntry>> ListSnapshotsAsync(
+        string providerId,
+        DateTimeOffset from,
+        DateTimeOffset to,
+        int limit,
+        CancellationToken ct = default)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(providerId);
+        await EnsureTableAsync(ct).ConfigureAwait(false);
+
+        const string sql = """
+            SELECT provider_id, digest, captured_at, epoch_timestamp
+            FROM replay.feed_snapshot_index
+            WHERE provider_id = @provider_id
+              AND captured_at >= @from
+              AND captured_at <= @to
+            ORDER BY captured_at ASC, digest ASC
+            LIMIT @limit;
+            """;
+
+        var effectiveLimit = limit <= 0 ? 1000 : limit;
+        var results = ImmutableArray.CreateBuilder<FeedSnapshotIndexEntry>(effectiveLimit);
+
+        await using var connection = await _dataSource.OpenConnectionAsync(ct).ConfigureAwait(false);
+        await using var command = new NpgsqlCommand(sql, connection);
+        command.Parameters.AddWithValue("provider_id", providerId);
+        command.Parameters.AddWithValue("from", from);
+        command.Parameters.AddWithValue("to", to);
+        command.Parameters.AddWithValue("limit", effectiveLimit);
+
+        await using var reader = await command.ExecuteReaderAsync(ct).ConfigureAwait(false);
+        while (await reader.ReadAsync(ct).ConfigureAwait(false))
+        {
+            results.Add(new FeedSnapshotIndexEntry
+            {
+                ProviderId = reader.GetString(0),
+                Digest = reader.GetString(1),
+                CapturedAt = reader.GetFieldValue<DateTimeOffset>(2),
+                EpochTimestamp = reader.GetFieldValue<DateTimeOffset>(3),
+            });
+        }
+
+        return results.ToImmutable();
+    }
+
+    public ValueTask DisposeAsync()
+    {
+        return _dataSource.DisposeAsync();
+    }
+
+    private async Task EnsureTableAsync(CancellationToken ct)
+    {
+        lock (_initGate)
+        {
+            if (_tableInitialized)
+            {
+                return;
+            }
+        }
+
+        const string ddl = """
+            CREATE SCHEMA IF NOT EXISTS replay;
+            CREATE TABLE IF NOT EXISTS replay.feed_snapshot_index (
+                provider_id TEXT NOT NULL,
+                digest TEXT NOT NULL,
+                captured_at TIMESTAMPTZ NOT NULL,
+                epoch_timestamp TIMESTAMPTZ NOT NULL,
+                PRIMARY KEY (provider_id, captured_at, digest)
+            );
+            CREATE INDEX IF NOT EXISTS idx_replay_snapshot_index_lookup
+                ON replay.feed_snapshot_index (provider_id, captured_at DESC, digest ASC);
+            """;
+
+        await using var connection = await _dataSource.OpenConnectionAsync(ct).ConfigureAwait(false);
+        await using var command = new NpgsqlCommand(ddl, connection);
+        await command.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
+
+        lock (_initGate)
+        {
+            _tableInitialized = true;
+        }
+    }
+}
+
+public sealed class SeedFsFeedSnapshotBlobStore : IFeedSnapshotBlobStore
+{
+    private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
+    {
+        WriteIndented = false,
+    };
+
+    private readonly string _rootPath;
+
+    public SeedFsFeedSnapshotBlobStore(string rootPath)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(rootPath);
+        _rootPath = Path.GetFullPath(rootPath);
+        Directory.CreateDirectory(_rootPath);
+    }
+
+    public async Task StoreAsync(FeedSnapshotBlob blob, CancellationToken ct = default)
+    {
+        ArgumentNullException.ThrowIfNull(blob);
+
+        var key = ToSafeKey(blob.Digest);
+        var contentPath = GetContentPath(key);
+        var metadataPath = GetMetadataPath(key);
+        Directory.CreateDirectory(Path.GetDirectoryName(contentPath)!);
+
+        await File.WriteAllBytesAsync(contentPath, blob.Content, ct).ConfigureAwait(false);
+
+        var metadata = FeedSnapshotBlobMetadata.FromBlob(blob);
+        var json = JsonSerializer.Serialize(metadata, JsonOptions);
+        await File.WriteAllTextAsync(metadataPath, json, ct).ConfigureAwait(false);
+    }
+
+    public async Task<FeedSnapshotBlob?> GetByDigestAsync(string digest, CancellationToken ct = default)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(digest);
+
+        var key = ToSafeKey(digest);
+        var contentPath = GetContentPath(key);
+        var metadataPath = GetMetadataPath(key);
+        if (!File.Exists(contentPath) || !File.Exists(metadataPath))
+        {
+            return null;
+        }
+
+        var content = await File.ReadAllBytesAsync(contentPath, ct).ConfigureAwait(false);
+        var json = await File.ReadAllTextAsync(metadataPath, ct).ConfigureAwait(false);
+        var metadata = JsonSerializer.Deserialize<FeedSnapshotBlobMetadata>(json, JsonOptions);
+        if (metadata is null)
+        {
+            return null;
+        }
+
+        return metadata.ToBlob(content);
+    }
+
+    public Task<bool> ExistsAsync(string digest, CancellationToken ct = default)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(digest);
+        var key = ToSafeKey(digest);
+        var exists = File.Exists(GetContentPath(key)) && File.Exists(GetMetadataPath(key));
+        return Task.FromResult(exists);
+    }
+
+    public Task DeleteAsync(string digest, CancellationToken ct = default)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(digest);
+        var key = ToSafeKey(digest);
+        var contentPath = GetContentPath(key);
+        var metadataPath = GetMetadataPath(key);
+
+        if (File.Exists(contentPath))
+        {
+            File.Delete(contentPath);
+        }
+
+        if (File.Exists(metadataPath))
+        {
+            File.Delete(metadataPath);
+        }
+
+        return Task.CompletedTask;
+    }
+
+    private string GetContentPath(string key) => Path.Combine(_rootPath, "snapshots", $"{key}.bin");
+
+    private string GetMetadataPath(string key) => Path.Combine(_rootPath, "snapshots", $"{key}.meta.json");
+
+    private static string ToSafeKey(string digest)
+    {
+        var value = digest.Trim();
+        value = value.Replace(':', '_').Replace('/', '_');
+        foreach (var invalid in Path.GetInvalidFileNameChars())
+        {
+            value = value.Replace(invalid, '_');
+        }
+
+        return string.IsNullOrWhiteSpace(value) ? "snapshot" : value;
+    }
+
+    private sealed record FeedSnapshotBlobMetadata(
+        string Digest,
+        string ProviderId,
+        string? ProviderName,
+        string? FeedType,
+        DateTimeOffset CapturedAt,
+        DateTimeOffset EpochTimestamp,
+        Dictionary<string, string> Metadata,
+        string ContentHash,
+        FeedSnapshotFormat Format)
+    {
+        public static FeedSnapshotBlobMetadata FromBlob(FeedSnapshotBlob blob)
+        {
+            return new FeedSnapshotBlobMetadata(
+                blob.Digest,
+                blob.ProviderId,
+                blob.ProviderName,
+                blob.FeedType,
+                blob.CapturedAt,
+                blob.EpochTimestamp,
+                blob.Metadata.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal),
+                blob.ContentHash,
+                blob.Format);
+        }
+
+        public FeedSnapshotBlob ToBlob(byte[] content)
+        {
+            return new FeedSnapshotBlob
+            {
+                Digest = Digest,
+                ProviderId = ProviderId,
+                ProviderName = ProviderName,
+                FeedType = FeedType,
+                Content = content,
+                CapturedAt = CapturedAt,
+                EpochTimestamp = EpochTimestamp,
+                Metadata = Metadata.ToImmutableDictionary(StringComparer.Ordinal),
+                ContentHash = ContentHash,
+                Format = Format
+            };
+        }
+    }
+}
--- a/src/Replay/StellaOps.Replay.WebService/StellaOps.Replay.WebService.csproj
+++ b/src/Replay/StellaOps.Replay.WebService/StellaOps.Replay.WebService.csproj
@@ -8,6 +8,7 @@
  </PropertyGroup>

  <ItemGroup>
+    <PackageReference Include="Npgsql" />
    <PackageReference Include="Serilog.AspNetCore" />
    <PackageReference Include="Serilog.Sinks.Console" />
  </ItemGroup>
--- a/src/Replay/StellaOps.Replay.WebService/TASKS.md
+++ b/src/Replay/StellaOps.Replay.WebService/TASKS.md
@@ -6,3 +6,4 @@ Source of truth: `docs/implplan/SPRINT_20260130_002_Tools_csproj_remediation_sol
 | --- | --- | --- |
 | REMED-05 | TODO | Remediation checklist: docs/implplan/audits/csproj-standards/remediation/checklists/src/Replay/StellaOps.Replay.WebService/StellaOps.Replay.WebService.md. |
 | REMED-06 | DONE | SOLID review notes captured for SPRINT_20260130_002. |
+| SPRINT-312-006 | DONE | Added Postgres snapshot index + seed-fs snapshot blob stores and wired storage-driver registration in webservice startup. |