up

2025-12-09 00:20:52 +02:00
parent 3d01bf9edc
commit bc0762e97d
261 changed files with 14033 additions and 4427 deletions
--- a/src/Excititor/StellaOps.Excititor.WebService/Program.cs
+++ b/src/Excititor/StellaOps.Excititor.WebService/Program.cs
@@ -27,28 +27,27 @@ using StellaOps.Excititor.Formats.CSAF;
 using StellaOps.Excititor.Formats.CycloneDX;
 using StellaOps.Excititor.Formats.OpenVEX;
 using StellaOps.Excititor.Policy;
-using StellaOps.Excititor.Storage.Mongo;
+using StellaOps.Excititor.Storage.Postgres;
 using StellaOps.Excititor.WebService.Endpoints;
 using StellaOps.Excititor.WebService.Extensions;
 using StellaOps.Excititor.WebService.Options;
 using StellaOps.Excititor.WebService.Services;
 using StellaOps.Excititor.Core.Aoc;
 using StellaOps.Excititor.WebService.Telemetry;
-using MongoDB.Driver;
-using MongoDB.Bson;
 using Microsoft.Extensions.Caching.Memory;
 using StellaOps.Excititor.WebService.Contracts;
 using System.Globalization;
 using StellaOps.Excititor.WebService.Graph;
+using StellaOps.Excititor.Core.Storage;

 var builder = WebApplication.CreateBuilder(args);
 var configuration = builder.Configuration;
 var services = builder.Services;
-services.AddOptions<VexMongoStorageOptions>()
-    .Bind(configuration.GetSection("Excititor:Storage:Mongo"))
+services.AddOptions<VexStorageOptions>()
+    .Bind(configuration.GetSection("Excititor:Storage"))
    .ValidateOnStart();

-services.AddExcititorMongoStorage();
+services.AddExcititorPostgresStorage(configuration);
 services.AddCsafNormalizer();
 services.AddCycloneDxNormalizer();
 services.AddOpenVexNormalizer();
@@ -147,7 +146,7 @@ app.UseObservabilityHeaders();

 app.MapGet("/excititor/status", async (HttpContext context,
    IEnumerable<IVexArtifactStore> artifactStores,
-    IOptions<VexMongoStorageOptions> mongoOptions,
+    IOptions<VexStorageOptions> mongoOptions,
    TimeProvider timeProvider) =>
 {
    var payload = new StatusResponse(
@@ -1260,7 +1259,7 @@ app.MapPost("/excititor/admin/backfill-statements", async (

 app.MapGet("/console/vex", async (
    HttpContext context,
-    IOptions<VexMongoStorageOptions> storageOptions,
+    IOptions<VexStorageOptions> storageOptions,
    IVexObservationQueryService queryService,
    ConsoleTelemetry telemetry,
    IMemoryCache cache,
@@ -1459,7 +1458,7 @@ var response = new GraphLinkoutsResponse(items, notFound);
 app.MapGet("/v1/graph/status", async (
    HttpContext context,
    [FromQuery(Name = "purl")] string[]? purls,
-    IOptions<VexMongoStorageOptions> storageOptions,
+    IOptions<VexStorageOptions> storageOptions,
    IOptions<GraphOptions> graphOptions,
    IVexObservationQueryService queryService,
    IMemoryCache cache,
@@ -1519,7 +1518,7 @@ app.MapGet("/v1/graph/overlays", async (
    HttpContext context,
    [FromQuery(Name = "purl")] string[]? purls,
    [FromQuery] bool includeJustifications,
-    IOptions<VexMongoStorageOptions> storageOptions,
+    IOptions<VexStorageOptions> storageOptions,
    IOptions<GraphOptions> graphOptions,
    IVexObservationQueryService queryService,
    IMemoryCache cache,
@@ -1580,7 +1579,7 @@ app.MapGet("/v1/graph/observations", async (
    [FromQuery] bool includeJustifications,
    [FromQuery] int? limitPerPurl,
    [FromQuery] string? cursor,
-    IOptions<VexMongoStorageOptions> storageOptions,
+    IOptions<VexStorageOptions> storageOptions,
    IOptions<GraphOptions> graphOptions,
    IVexObservationQueryService queryService,
    CancellationToken cancellationToken) =>
@@ -1638,7 +1637,7 @@ app.MapPost("/ingest/vex", async (
    HttpContext context,
    VexIngestRequest request,
    IVexRawStore rawStore,
-    IOptions<VexMongoStorageOptions> storageOptions,
+    IOptions<VexStorageOptions> storageOptions,
    TimeProvider timeProvider,
    ILogger<Program> logger,
    CancellationToken cancellationToken) =>
@@ -1692,8 +1691,8 @@ app.MapPost("/ingest/vex", async (

 app.MapGet("/vex/raw", async (
    HttpContext context,
-    IMongoDatabase database,
-    IOptions<VexMongoStorageOptions> storageOptions,
+    IVexRawStore rawStore,
+    IOptions<VexStorageOptions> storageOptions,
    CancellationToken cancellationToken) =>
 {
    var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
@@ -1702,132 +1701,69 @@ app.MapGet("/vex/raw", async (
        return scopeResult;
    }

-    if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out _, out var tenantError))
+    if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out var tenant, out var tenantError))
    {
        return tenantError;
    }

-    var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
    var query = context.Request.Query;
-    var filters = new List<FilterDefinition<BsonDocument>>();
-    var builder = Builders<BsonDocument>.Filter;
+    var providerFilter = BuildStringFilterSet(query["providerId"]);
+    var digestFilter = BuildStringFilterSet(query["digest"]);
+    var formatFilter = query.TryGetValue("format", out var formats)
+        ? formats
+            .Where(static f => !string.IsNullOrWhiteSpace(f))
+            .Select(static f => Enum.TryParse<VexDocumentFormat>(f, true, out var parsed) ? parsed : VexDocumentFormat.Unknown)
+            .Where(static f => f != VexDocumentFormat.Unknown)
+            .ToArray()
+        : Array.Empty<VexDocumentFormat>();

-    if (query.TryGetValue("providerId", out var providerValues))
-    {
-        var providers = providerValues
-            .Where(static value => !string.IsNullOrWhiteSpace(value))
-            .Select(static value => value!.Trim())
-            .ToArray();
-        if (providers.Length > 0)
-        {
-            filters.Add(builder.In("ProviderId", providers));
-        }
-    }
-
-    if (query.TryGetValue("digest", out var digestValues))
-    {
-        var digests = digestValues
-            .Where(static value => !string.IsNullOrWhiteSpace(value))
-            .Select(static value => value!.Trim())
-            .ToArray();
-        if (digests.Length > 0)
-        {
-            filters.Add(builder.In("Digest", digests));
-        }
-    }
-
-    if (query.TryGetValue("format", out var formatValues))
-    {
-        var formats = formatValues
-            .Where(static value => !string.IsNullOrWhiteSpace(value))
-            .Select(static value => value!.Trim().ToLowerInvariant())
-            .ToArray();
-        if (formats.Length > 0)
-        {
-            filters.Add(builder.In("Format", formats));
-        }
-    }
-
-    if (query.TryGetValue("since", out var sinceValues) && DateTimeOffset.TryParse(sinceValues.FirstOrDefault(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var sinceValue))
-    {
-        filters.Add(builder.Gte("RetrievedAt", sinceValue.UtcDateTime));
-    }
+    var since = ParseSinceTimestamp(query["since"]);

    var cursorToken = query.TryGetValue("cursor", out var cursorValues) ? cursorValues.FirstOrDefault() : null;
-    DateTime? cursorTimestamp = null;
-    string? cursorDigest = null;
-    if (!string.IsNullOrWhiteSpace(cursorToken) && TryDecodeCursor(cursorToken, out var cursorTime, out var cursorId))
+    VexRawCursor? cursor = null;
+    if (!string.IsNullOrWhiteSpace(cursorToken) &&
+        TryDecodeCursor(cursorToken, out var cursorTime, out var cursorId))
    {
-        cursorTimestamp = cursorTime.UtcDateTime;
-        cursorDigest = cursorId;
+        cursor = new VexRawCursor(cursorTime, cursorId);
    }

-    if (cursorTimestamp is not null && cursorDigest is not null)
-    {
-        var ltTime = builder.Lt("RetrievedAt", cursorTimestamp.Value);
-        var eqTimeLtDigest = builder.And(
-            builder.Eq("RetrievedAt", cursorTimestamp.Value),
-            builder.Lt("Digest", cursorDigest));
-        filters.Add(builder.Or(ltTime, eqTimeLtDigest));
-    }
+    var limit = ResolveLimit(query["limit"], defaultValue: 50, min: 1, max: 200);

-    var limit = 50;
-    if (query.TryGetValue("limit", out var limitValues) && int.TryParse(limitValues.FirstOrDefault(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var requestedLimit))
-    {
-        limit = Math.Clamp(requestedLimit, 1, 200);
-    }
+    var page = await rawStore.QueryAsync(
+        new VexRawQuery(
+            tenant,
+            providerFilter,
+            digestFilter,
+            formatFilter,
+            since,
+            Until: null,
+            cursor,
+            limit),
+        cancellationToken).ConfigureAwait(false);

-    var filter = filters.Count == 0 ? builder.Empty : builder.And(filters);
-    var sort = Builders<BsonDocument>.Sort.Descending("RetrievedAt").Descending("Digest");
-    var documents = await collection
-        .Find(filter)
-        .Sort(sort)
-        .Limit(limit)
-        .Project(Builders<BsonDocument>.Projection.Include("Digest").Include("ProviderId").Include("Format").Include("SourceUri").Include("RetrievedAt").Include("Metadata").Include("GridFsObjectId"))
-        .ToListAsync(cancellationToken)
-        .ConfigureAwait(false);
+    var summaries = page.Items
+        .Select(summary => new VexRawSummaryResponse(
+            summary.Digest,
+            summary.ProviderId,
+            summary.Format.ToString().ToLowerInvariant(),
+            summary.SourceUri.ToString(),
+            summary.RetrievedAt,
+            summary.InlineContent,
+            summary.Metadata))
+        .ToList();

-    var summaries = new List<VexRawSummaryResponse>(documents.Count);
-    foreach (var document in documents)
-    {
-        var digest = document.TryGetValue("Digest", out var digestValue) && digestValue.IsString ? digestValue.AsString : string.Empty;
-        var providerId = document.TryGetValue("ProviderId", out var providerValue) && providerValue.IsString ? providerValue.AsString : string.Empty;
-        var format = document.TryGetValue("Format", out var formatValue) && formatValue.IsString ? formatValue.AsString : string.Empty;
-        var sourceUri = document.TryGetValue("SourceUri", out var sourceValue) && sourceValue.IsString ? sourceValue.AsString : string.Empty;
-        var retrievedAt = document.TryGetValue("RetrievedAt", out var retrievedValue) && retrievedValue is BsonDateTime bsonDate
-            ? bsonDate.ToUniversalTime()
-            : DateTime.UtcNow;
-        var metadata = ReadMetadata(document.TryGetValue("Metadata", out var metadataValue) ? metadataValue : BsonNull.Value);
-        var inlineContent = !document.TryGetValue("GridFsObjectId", out var gridId) || gridId.IsBsonNull || (gridId.IsString && string.IsNullOrWhiteSpace(gridId.AsString));
+    var nextCursor = page.NextCursor is null
+        ? null
+        : EncodeCursor(page.NextCursor.RetrievedAt.UtcDateTime, page.NextCursor.Digest);

-        summaries.Add(new VexRawSummaryResponse(
-            digest,
-            providerId,
-            format,
-            sourceUri,
-            new DateTimeOffset(retrievedAt),
-            inlineContent,
-            metadata));
-    }
-
-    var hasMore = documents.Count == limit;
-    string? nextCursor = null;
-    if (hasMore && documents.Count > 0)
-    {
-        var last = documents[^1];
-        var lastTime = last.GetValue("RetrievedAt", BsonNull.Value).ToUniversalTime();
-        var lastDigest = last.GetValue("Digest", BsonNull.Value).AsString;
-        nextCursor = EncodeCursor(lastTime, lastDigest);
-    }
-
-    return Results.Json(new VexRawListResponse(summaries, nextCursor, hasMore));
+    return Results.Json(new VexRawListResponse(summaries, nextCursor, page.HasMore));
 });

 app.MapGet("/vex/raw/{digest}", async (
    string digest,
    HttpContext context,
    IVexRawStore rawStore,
-    IOptions<VexMongoStorageOptions> storageOptions,
+    IOptions<VexStorageOptions> storageOptions,
    CancellationToken cancellationToken) =>
 {
    var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
@@ -1861,7 +1797,7 @@ app.MapGet("/vex/raw/{digest}/provenance", async (
    string digest,
    HttpContext context,
    IVexRawStore rawStore,
-    IOptions<VexMongoStorageOptions> storageOptions,
+    IOptions<VexStorageOptions> storageOptions,
    CancellationToken cancellationToken) =>
 {
    var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
@@ -1901,7 +1837,7 @@ app.MapGet("/v1/vex/observations/{vulnerabilityId}/{productKey}", async (
    string vulnerabilityId,
    string productKey,
    [FromServices] IVexObservationProjectionService projectionService,
-    [FromServices] IOptions<VexMongoStorageOptions> storageOptions,
+    [FromServices] IOptions<VexStorageOptions> storageOptions,
    CancellationToken cancellationToken) =>
 {
    var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
@@ -1977,7 +1913,7 @@ app.MapGet("/v1/vex/observations/{vulnerabilityId}/{productKey}", async (
 app.MapGet("/v1/vex/evidence/chunks", async (
    HttpContext context,
    [FromServices] IVexEvidenceChunkService chunkService,
-    [FromServices] IOptions<VexMongoStorageOptions> storageOptions,
+    [FromServices] IOptions<VexStorageOptions> storageOptions,
    [FromServices] ChunkTelemetry chunkTelemetry,
    [FromServices] ILogger<VexEvidenceChunkRequest> logger,
    [FromServices] TimeProvider timeProvider,
@@ -2083,10 +2019,9 @@ app.MapGet("/v1/vex/evidence/chunks", async (
 app.MapPost("/aoc/verify", async (
    HttpContext context,
    VexAocVerifyRequest? request,
-    IMongoDatabase database,
    IVexRawStore rawStore,
    IVexRawWriteGuard guard,
-    IOptions<VexMongoStorageOptions> storageOptions,
+    IOptions<VexStorageOptions> storageOptions,
    TimeProvider timeProvider,
    CancellationToken cancellationToken) =>
 {
@@ -2119,33 +2054,26 @@ app.MapPost("/aoc/verify", async (
        .Select(static value => value!.Trim())
        .ToArray();

-    var builder = Builders<BsonDocument>.Filter;
-    var filter = builder.And(
-        builder.Gte("RetrievedAt", since),
-        builder.Lte("RetrievedAt", until));
-
-    if (sources is { Length: > 0 })
-    {
-        filter &= builder.In("ProviderId", sources);
-    }
-
-    var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
-    var digests = await collection
-        .Find(filter)
-        .Sort(Builders<BsonDocument>.Sort.Descending("RetrievedAt"))
-        .Limit(limit)
-        .Project(Builders<BsonDocument>.Projection.Include("Digest").Include("RetrievedAt").Include("ProviderId"))
-        .ToListAsync(cancellationToken)
-        .ConfigureAwait(false);
+    var page = await rawStore.QueryAsync(
+        new VexRawQuery(
+            tenant,
+            sources ?? Array.Empty<string>(),
+            Array.Empty<string>(),
+            Array.Empty<VexDocumentFormat>(),
+            since: new DateTimeOffset(since, TimeSpan.Zero),
+            until: new DateTimeOffset(until, TimeSpan.Zero),
+            cursor: null,
+            limit),
+        cancellationToken).ConfigureAwait(false);

    var checkedCount = 0;
    var violationMap = new Dictionary<string, (int Count, List<VexAocVerifyViolationExample> Examples)>(StringComparer.OrdinalIgnoreCase);
    const int MaxExamplesPerCode = 5;

-    foreach (var digestDocument in digests)
+    foreach (var item in page.Items)
    {
-        var digestValue = digestDocument.GetValue("Digest", BsonNull.Value).AsString;
-        var provider = digestDocument.GetValue("ProviderId", BsonNull.Value).AsString;
+        var digestValue = item.Digest;
+        var provider = item.ProviderId;

        var domainDocument = await rawStore.FindByDigestAsync(digestValue, cancellationToken).ConfigureAwait(false);
        if (domainDocument is null)
@@ -2202,7 +2130,7 @@ app.MapPost("/aoc/verify", async (
        new VexAocVerifyChecked(0, checkedCount),
        violations,
        new VexAocVerifyMetrics(checkedCount, violations.Sum(v => v.Count)),
-        digests.Count == limit);
+        page.HasMore);

    return Results.Json(response);
 });
@@ -2225,7 +2153,7 @@ app.MapGet("/obs/excititor/health", async (
 // VEX timeline SSE (WEB-OBS-52-001)
 app.MapGet("/obs/excititor/timeline", async (
    HttpContext context,
-    IOptions<VexMongoStorageOptions> storageOptions,
+    IOptions<VexStorageOptions> storageOptions,
    [FromServices] IVexTimelineEventStore timelineStore,
    TimeProvider timeProvider,
    ILoggerFactory loggerFactory,