up

src/Concelier/StellaOps.Concelier.WebService/Program.cs

@@ -26,6 +26,7 @@ using StellaOps.Concelier.Core.Events;
 using StellaOps.Concelier.Core.Jobs;
 using StellaOps.Concelier.Core.Observations;
 using StellaOps.Concelier.Core.Linksets;
+using StellaOps.Concelier.Core.Diagnostics;
 using StellaOps.Concelier.Models;
 using StellaOps.Concelier.WebService.Diagnostics;
 using ServiceStatus = StellaOps.Concelier.WebService.Diagnostics.ServiceStatus;
@@ -54,9 +55,6 @@ using StellaOps.Concelier.Core.Aoc;
 using StellaOps.Concelier.Core.Raw;
 using StellaOps.Concelier.RawModels;
 using StellaOps.Concelier.Storage.Postgres;
-using StellaOps.Concelier.Storage.Mongo;
-using StellaOps.Concelier.Storage.Mongo.Advisories;
-using StellaOps.Concelier.Storage.Mongo.Aliases;
 using StellaOps.Concelier.Core.Attestation;
 using StellaOps.Concelier.Core.Signals;
 using AttestationClaims = StellaOps.Concelier.Core.Attestation.AttestationClaims;
@@ -64,8 +62,10 @@ using StellaOps.Concelier.Core.Orchestration;
 using System.Diagnostics.Metrics;
 using StellaOps.Concelier.Models.Observations;
 using StellaOps.Aoc.AspNetCore.Results;
-using StellaOps.Provenance.Mongo;
 using HttpResults = Microsoft.AspNetCore.Http.Results;
+using StellaOps.Concelier.Storage.Mongo.Advisories;
+using StellaOps.Concelier.Storage.Mongo.Aliases;
+using StellaOps.Provenance.Mongo;
 
 namespace StellaOps.Concelier.WebService
 {
@@ -91,9 +91,10 @@ builder.Host.ConfigureAppConfiguration((context, cfg) =>
     {
         cfg.AddInMemoryCollection(new Dictionary<string, string?>
         {
-            {"Concelier:Storage:Dsn", Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN") ?? "mongodb://localhost:27017/test-health"},
-            {"Concelier:Storage:Driver", "mongo"},
-            {"Concelier:Storage:CommandTimeoutSeconds", "30"},
+            {"Concelier:PostgresStorage:Enabled", "true"},
+            {"Concelier:PostgresStorage:ConnectionString", Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN") ?? "Host=localhost;Port=5432;Database=concelier_test;Username=postgres;Password=postgres"},
+            {"Concelier:PostgresStorage:CommandTimeoutSeconds", "30"},
+            {"Concelier:PostgresStorage:SchemaName", "vuln"},
             {"Concelier:Telemetry:Enabled", "false"}
         });
     }
@@ -125,11 +126,12 @@ if (builder.Environment.IsEnvironment("Testing"))
         #pragma warning restore ASP0000
         concelierOptions = tempProvider.GetService<IOptions<ConcelierOptions>>()?.Value ?? new ConcelierOptions
         {
-            Storage = new ConcelierOptions.StorageOptions
+            PostgresStorage = new ConcelierOptions.PostgresStorageOptions
             {
-                Dsn = Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN") ?? "mongodb://localhost:27017/test-health",
-                Driver = "mongo",
-                CommandTimeoutSeconds = 30
+                Enabled = true,
+                ConnectionString = Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN") ?? "Host=localhost;Port=5432;Database=concelier_test;Username=postgres;Password=postgres",
+                CommandTimeoutSeconds = 30,
+                SchemaName = "vuln"
             },
             Telemetry = new ConcelierOptions.TelemetryOptions
             {
@@ -137,10 +139,18 @@ if (builder.Environment.IsEnvironment("Testing"))
             }
         };
 
-        concelierOptions.Storage ??= new ConcelierOptions.StorageOptions();
-        concelierOptions.Storage.Dsn = Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN") ?? "mongodb://localhost:27017/orch-tests";
-        concelierOptions.Storage.Driver = "mongo";
-        concelierOptions.Storage.CommandTimeoutSeconds = concelierOptions.Storage.CommandTimeoutSeconds <= 0 ? 30 : concelierOptions.Storage.CommandTimeoutSeconds;
+        concelierOptions.PostgresStorage ??= new ConcelierOptions.PostgresStorageOptions
+        {
+            Enabled = true,
+            ConnectionString = Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN") ?? "Host=localhost;Port=5432;Database=concelier_test;Username=postgres;Password=postgres",
+            CommandTimeoutSeconds = 30,
+            SchemaName = "vuln"
+        };
+
+        if (string.IsNullOrWhiteSpace(concelierOptions.PostgresStorage.ConnectionString))
+        {
+            concelierOptions.PostgresStorage.ConnectionString = Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN") ?? string.Empty;
+        }
 
         ConcelierOptionsPostConfigure.Apply(concelierOptions, contentRootPath);
         // Skip validation in Testing to allow factory-provided wiring.
@@ -149,10 +159,21 @@ else
 {
     concelierOptions = builder.Configuration.BindOptions<ConcelierOptions>(postConfigure: (opts, _) =>
     {
-        var testDsn = Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN");
-        if (string.IsNullOrWhiteSpace(opts.Storage.Dsn) && !string.IsNullOrWhiteSpace(testDsn))
+        var testDsn = Environment.GetEnvironmentVariable("CONCELIER_POSTGRES_DSN")
+                       ?? Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN");
+
+        opts.PostgresStorage ??= new ConcelierOptions.PostgresStorageOptions
         {
-            opts.Storage.Dsn = testDsn;
+            Enabled = !string.IsNullOrWhiteSpace(testDsn),
+            ConnectionString = testDsn ?? string.Empty,
+            SchemaName = "vuln",
+            CommandTimeoutSeconds = 30
+        };
+
+        if (string.IsNullOrWhiteSpace(opts.PostgresStorage.ConnectionString) && !string.IsNullOrWhiteSpace(testDsn))
+        {
+            opts.PostgresStorage.ConnectionString = testDsn;
+            opts.PostgresStorage.Enabled = true;
         }
 
         ConcelierOptionsPostConfigure.Apply(opts, contentRootPath);
@@ -179,24 +200,26 @@ builder.Services.AddSingleton<MirrorFileLocator>();
 
 var isTesting = builder.Environment.IsEnvironment("Testing");
 
-// Add PostgreSQL storage for LNM linkset cache if configured.
-// This provides a PostgreSQL-backed implementation of IAdvisoryLinksetStore for the read-through cache.
-if (concelierOptions.PostgresStorage is { Enabled: true } postgresOptions)
+// Add PostgreSQL storage for all Concelier persistence.
+var postgresOptions = concelierOptions.PostgresStorage ?? throw new InvalidOperationException("PostgreSQL storage must be configured.");
+if (!postgresOptions.Enabled)
 {
-    builder.Services.AddConcelierPostgresStorage(pgOptions =>
-    {
-        pgOptions.ConnectionString = postgresOptions.ConnectionString;
-        pgOptions.CommandTimeoutSeconds = postgresOptions.CommandTimeoutSeconds;
-        pgOptions.MaxPoolSize = postgresOptions.MaxPoolSize;
-        pgOptions.MinPoolSize = postgresOptions.MinPoolSize;
-        pgOptions.ConnectionIdleLifetimeSeconds = postgresOptions.ConnectionIdleLifetimeSeconds;
-        pgOptions.Pooling = postgresOptions.Pooling;
-        pgOptions.SchemaName = postgresOptions.SchemaName;
-        pgOptions.AutoMigrate = postgresOptions.AutoMigrate;
-        pgOptions.MigrationsPath = postgresOptions.MigrationsPath;
-    });
+    throw new InvalidOperationException("PostgreSQL storage must be enabled.");
 }
 
+builder.Services.AddConcelierPostgresStorage(pgOptions =>
+{
+    pgOptions.ConnectionString = postgresOptions.ConnectionString;
+    pgOptions.CommandTimeoutSeconds = postgresOptions.CommandTimeoutSeconds;
+    pgOptions.MaxPoolSize = postgresOptions.MaxPoolSize;
+    pgOptions.MinPoolSize = postgresOptions.MinPoolSize;
+    pgOptions.ConnectionIdleLifetimeSeconds = postgresOptions.ConnectionIdleLifetimeSeconds;
+    pgOptions.Pooling = postgresOptions.Pooling;
+    pgOptions.SchemaName = postgresOptions.SchemaName;
+    pgOptions.AutoMigrate = postgresOptions.AutoMigrate;
+    pgOptions.MigrationsPath = postgresOptions.MigrationsPath;
+});
+
 builder.Services.AddOptions<AdvisoryObservationEventPublisherOptions>()
     .Bind(builder.Configuration.GetSection("advisoryObservationEvents"))
     .PostConfigure(options =>
@@ -1039,9 +1062,12 @@ var advisoryIngestEndpoint = app.MapPost("/ingest/advisory", async (
         return Problem(context, "Invalid advisory payload", StatusCodes.Status400BadRequest, ProblemTypes.Validation, ex.Message);
     }
 
+    var chunkStopwatch = Stopwatch.StartNew();
+
     try
     {
         var result = await rawService.IngestAsync(document, cancellationToken).ConfigureAwait(false);
+        chunkStopwatch.Stop();
 
         var response = new AdvisoryIngestResponse(
             result.Record.Id,
@@ -1065,10 +1091,21 @@ var advisoryIngestEndpoint = app.MapPost("/ingest/advisory", async (
                 ingestRequest.Source.Vendor ?? "(unknown)",
                 result.Inserted ? "inserted" : "duplicate"));
 
+        var telemetrySource = ingestRequest.Source.Vendor ?? "(unknown)";
+        var (_, _, conflicts) = AdvisoryLinksetNormalization.FromRawLinksetWithConfidence(document.Linkset, providedConfidence: null);
+        var collisionCount = VulnExplorerTelemetry.CountAliasCollisions(conflicts);
+        VulnExplorerTelemetry.RecordIdentifierCollisions(tenant, telemetrySource, collisionCount);
+        VulnExplorerTelemetry.RecordChunkLatency(tenant, telemetrySource, chunkStopwatch.Elapsed);
+        if (VulnExplorerTelemetry.IsWithdrawn(document.Content.Raw))
+        {
+            VulnExplorerTelemetry.RecordWithdrawnStatement(tenant, telemetrySource);
+        }
+
         return JsonResult(response, statusCode);
     }
     catch (ConcelierAocGuardException guardException)
     {
+        chunkStopwatch.Stop();
         logger.LogWarning(
             guardException,
             "AOC guard rejected advisory ingest tenant={Tenant} upstream={UpstreamId} requestHash={RequestHash} documentHash={DocumentHash} codes={Codes}",
@@ -2115,6 +2152,12 @@ var advisoryChunksEndpoint = app.MapGet("/advisories/{advisoryKey}/chunks", asyn
         buildResult.Response.Entries.Count,
         duration,
         guardrailCounts));
+    VulnExplorerTelemetry.RecordChunkRequest(
+        tenant!,
+        result: "ok",
+        cacheHit,
+        buildResult.Response.Entries.Count,
+        duration.TotalMilliseconds);
 
     return JsonResult(buildResult.Response);
 });
@@ -3269,7 +3312,7 @@ void ApplyNoCache(HttpResponse response)
     response.Headers["Expires"] = "0";
 }
 
-await InitializeMongoAsync(app);
+await InitializePostgresAsync(app);
 
 app.MapGet("/health", ([FromServices] IOptions<ConcelierOptions> opts, [FromServices] StellaOps.Concelier.WebService.Diagnostics.ServiceStatus status, HttpContext context) =>
 {
@@ -3278,11 +3321,12 @@ app.MapGet("/health", ([FromServices] IOptions<ConcelierOptions> opts, [FromServ
     var snapshot = status.CreateSnapshot();
     var uptimeSeconds = Math.Max((snapshot.CapturedAt - snapshot.StartedAt).TotalSeconds, 0d);
 
-    var storage = new StorageBootstrapHealth(
-        Driver: opts.Value.Storage.Driver,
-        Completed: snapshot.BootstrapCompletedAt is not null,
-        CompletedAt: snapshot.BootstrapCompletedAt,
-        DurationMs: snapshot.BootstrapDuration?.TotalMilliseconds);
+    var storage = new StorageHealth(
+        Backend: "postgres",
+        Ready: snapshot.LastReadySucceeded,
+        CheckedAt: snapshot.LastReadyCheckAt,
+        LatencyMs: snapshot.LastStorageLatency?.TotalMilliseconds,
+        Error: snapshot.LastStorageError);
 
     var telemetry = new TelemetryHealth(
         Enabled: opts.Value.Telemetry.Enabled,
@@ -3300,24 +3344,32 @@ app.MapGet("/health", ([FromServices] IOptions<ConcelierOptions> opts, [FromServ
     return JsonResult(response);
 });
 
-app.MapGet("/ready", ([FromServices] StellaOps.Concelier.WebService.Diagnostics.ServiceStatus status, HttpContext context) =>
+app.MapGet("/ready", async (
+    [FromServices] StellaOps.Concelier.WebService.Diagnostics.ServiceStatus status,
+    [FromServices] ConcelierDataSource dataSource,
+    HttpContext context,
+    CancellationToken cancellationToken) =>
 {
     ApplyNoCache(context.Response);
 
+    var (ready, latency, error) = await CheckPostgresAsync(dataSource, cancellationToken).ConfigureAwait(false);
+    status.RecordStorageCheck(ready, latency, error);
+
     var snapshot = status.CreateSnapshot();
     var uptimeSeconds = Math.Max((snapshot.CapturedAt - snapshot.StartedAt).TotalSeconds, 0d);
 
-    var mongo = new MongoReadyHealth(
-        Status: "bypassed",
-        LatencyMs: null,
+    var storage = new StorageHealth(
+        Backend: "postgres",
+        Ready: ready,
         CheckedAt: snapshot.LastReadyCheckAt,
-        Error: "mongo disabled");
+        LatencyMs: latency.TotalMilliseconds,
+        Error: error);
 
     var response = new ReadyDocument(
-        Status: "ready",
+        Status: ready ? "ready" : "degraded",
         StartedAt: snapshot.StartedAt,
         UptimeSeconds: uptimeSeconds,
-        Mongo: mongo);
+        Storage: storage);
 
     return JsonResult(response);
 });
@@ -4019,9 +4071,54 @@ static SignalsSymbolSetResponse ToSymbolSetResponse(AffectedSymbolSet symbolSet)
     return pluginOptions;
 }
 
-static async Task InitializeMongoAsync(WebApplication app)
+static async Task InitializePostgresAsync(WebApplication app)
 {
-    await Task.CompletedTask;
+    var dataSource = app.Services.GetService<ConcelierDataSource>();
+    var status = app.Services.GetRequiredService<StellaOps.Concelier.WebService.Diagnostics.ServiceStatus>();
+
+    if (dataSource is null)
+    {
+        status.RecordStorageCheck(false, TimeSpan.Zero, "PostgreSQL storage not configured");
+        return;
+    }
+
+    var stopwatch = Stopwatch.StartNew();
+    try
+    {
+        var (ready, latency, error) = await CheckPostgresAsync(dataSource, CancellationToken.None).ConfigureAwait(false);
+        stopwatch.Stop();
+        status.RecordStorageCheck(ready, latency, error);
+        if (ready)
+        {
+            status.MarkBootstrapCompleted(latency);
+        }
+    }
+    catch (Exception ex)
+    {
+        stopwatch.Stop();
+        status.RecordStorageCheck(false, stopwatch.Elapsed, ex.Message);
+    }
+}
+
+static async Task<(bool Ready, TimeSpan Latency, string? Error)> CheckPostgresAsync(
+    ConcelierDataSource dataSource,
+    CancellationToken cancellationToken)
+{
+    var stopwatch = Stopwatch.StartNew();
+    try
+    {
+        await using var connection = await dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
+        await using var command = connection.CreateCommand();
+        command.CommandText = "select 1";
+        _ = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
+        stopwatch.Stop();
+        return (true, stopwatch.Elapsed, null);
+    }
+    catch (Exception ex)
+    {
+        stopwatch.Stop();
+        return (false, stopwatch.Elapsed, ex.Message);
+    }
 }
 
 }