Implement ledger metrics for observability and add tests for Ruby packages endpoints

- Added `LedgerMetrics` class to record write latency and total events for ledger operations.
- Created comprehensive tests for Ruby packages endpoints, covering scenarios for missing inventory, successful retrieval, and identifier handling.
- Introduced `TestSurfaceSecretsScope` for managing environment variables during tests.
- Developed `ProvenanceMongoExtensions` for attaching DSSE provenance and trust information to event documents.
- Implemented `EventProvenanceWriter` and `EventWriter` classes for managing event provenance in MongoDB.
- Established MongoDB indexes for efficient querying of events based on provenance and trust.
- Added models and JSON parsing logic for DSSE provenance and trust information.

src/Concelier/StellaOps.Concelier.WebService/Program.cs

@@ -24,9 +24,9 @@ using MongoDB.Bson;
 using MongoDB.Driver;
 using StellaOps.Concelier.Core.Events;
 using StellaOps.Concelier.Core.Jobs;
-using StellaOps.Concelier.Storage.Mongo;
 using StellaOps.Concelier.Core.Observations;
 using StellaOps.Concelier.Core.Linksets;
+using StellaOps.Concelier.Models;
 using StellaOps.Concelier.WebService.Diagnostics;
 using Serilog;
 using StellaOps.Concelier.Merge;
@@ -50,6 +50,10 @@ using StellaOps.Concelier.WebService.Contracts;
 using StellaOps.Concelier.Core.Aoc;
 using StellaOps.Concelier.Core.Raw;
 using StellaOps.Concelier.RawModels;
+using StellaOps.Concelier.Storage.Mongo;
+using StellaOps.Concelier.Storage.Mongo.Advisories;
+using StellaOps.Concelier.Storage.Mongo.Aliases;
+using StellaOps.Provenance.Mongo;
 
 var builder = WebApplication.CreateBuilder(args);
 
@@ -812,6 +816,8 @@ var advisoryChunksEndpoint = app.MapGet("/advisories/{advisoryKey}/chunks", asyn
     [FromServices] IAdvisoryObservationQueryService observationService,
     [FromServices] AdvisoryChunkBuilder chunkBuilder,
     [FromServices] IAdvisoryChunkCache chunkCache,
+    [FromServices] IAdvisoryStore advisoryStore,
+    [FromServices] IAliasStore aliasStore,
     [FromServices] IAdvisoryAiTelemetry telemetry,
     [FromServices] TimeProvider timeProvider,
     CancellationToken cancellationToken) =>
@@ -854,21 +860,37 @@ var advisoryChunksEndpoint = app.MapGet("/advisories/{advisoryKey}/chunks", asyn
     var sectionFilter = BuildFilterSet(context.Request.Query["section"]);
     var formatFilter = BuildFilterSet(context.Request.Query["format"]);
 
+    var resolution = await ResolveAdvisoryAsync(
+        normalizedKey,
+        advisoryStore,
+        aliasStore,
+        cancellationToken).ConfigureAwait(false);
+    if (resolution is null)
+    {
+        telemetry.TrackChunkFailure(tenant, normalizedKey, "advisory_not_found", "not_found");
+        return Problem(context, "Advisory not found", StatusCodes.Status404NotFound, ProblemTypes.NotFound, $"No advisory found for {normalizedKey}.");
+    }
+
+    var (advisory, aliasList, fingerprint) = resolution.Value;
+    var aliasCandidates = aliasList.IsDefaultOrEmpty
+        ? ImmutableArray.Create(advisory.AdvisoryKey)
+        : aliasList;
+
     var queryOptions = new AdvisoryObservationQueryOptions(
         tenant,
-        aliases: new[] { normalizedKey },
+        aliases: aliasCandidates,
         limit: observationLimit);
 
     var observationResult = await observationService.QueryAsync(queryOptions, cancellationToken).ConfigureAwait(false);
     if (observationResult.Observations.IsDefaultOrEmpty || observationResult.Observations.Length == 0)
     {
-        telemetry.TrackChunkFailure(tenant, normalizedKey, "advisory_not_found", "not_found");
-        return Problem(context, "Advisory not found", StatusCodes.Status404NotFound, ProblemTypes.NotFound, $"No observations available for {normalizedKey}.");
+        telemetry.TrackChunkFailure(tenant, advisory.AdvisoryKey, "advisory_not_found", "not_found");
+        return Problem(context, "Advisory not found", StatusCodes.Status404NotFound, ProblemTypes.NotFound, $"No observations available for {advisory.AdvisoryKey}.");
     }
 
     var observations = observationResult.Observations.ToArray();
     var buildOptions = new AdvisoryChunkBuildOptions(
-        normalizedKey,
+        advisory.AdvisoryKey,
         chunkLimit,
         observationLimit,
         sectionFilter,
@@ -884,7 +906,7 @@ var advisoryChunksEndpoint = app.MapGet("/advisories/{advisoryKey}/chunks", asyn
 
     if (cacheDuration > TimeSpan.Zero)
     {
-        var cacheKey = AdvisoryChunkCacheKey.Create(tenant, normalizedKey, buildOptions, observations);
+        var cacheKey = AdvisoryChunkCacheKey.Create(tenant, advisory.AdvisoryKey, buildOptions, observations, fingerprint);
         if (chunkCache.TryGet(cacheKey, out var cachedResult))
         {
             buildResult = cachedResult;
@@ -892,13 +914,13 @@ var advisoryChunksEndpoint = app.MapGet("/advisories/{advisoryKey}/chunks", asyn
         }
         else
         {
-            buildResult = chunkBuilder.Build(buildOptions, observations);
+            buildResult = chunkBuilder.Build(buildOptions, advisory, observations);
             chunkCache.Set(cacheKey, buildResult, cacheDuration);
         }
     }
     else
     {
-        buildResult = chunkBuilder.Build(buildOptions, observations);
+        buildResult = chunkBuilder.Build(buildOptions, advisory, observations);
     }
 
     var duration = timeProvider.GetElapsedTime(requestStart);
@@ -907,13 +929,13 @@ var advisoryChunksEndpoint = app.MapGet("/advisories/{advisoryKey}/chunks", asyn
 
     telemetry.TrackChunkResult(new AdvisoryAiChunkRequestTelemetry(
         tenant,
-        normalizedKey,
+        advisory.AdvisoryKey,
         "ok",
         buildResult.Response.Truncated,
         cacheHit,
         observations.Length,
         buildResult.Telemetry.SourceCount,
-        buildResult.Response.Chunks.Count,
+        buildResult.Response.Entries.Count,
         duration,
         guardrailCounts));
 
@@ -1055,6 +1077,52 @@ app.MapGet("/concelier/advisories/{vulnerabilityKey}/replay", async (
     return JsonResult(response);
 });
 
+var statementProvenanceEndpoint = app.MapPost("/events/statements/{statementId:guid}/provenance", async (
+    Guid statementId,
+    HttpContext context,
+    [FromServices] IAdvisoryEventLog eventLog,
+    CancellationToken cancellationToken) =>
+{
+    if (!TryResolveTenant(context, requireHeader: true, out var tenant, out var tenantError))
+    {
+        return tenantError;
+    }
+
+    var authorizationError = EnsureTenantAuthorized(context, tenant);
+    if (authorizationError is not null)
+    {
+        return authorizationError;
+    }
+
+    try
+    {
+        using var document = await JsonDocument.ParseAsync(context.Request.Body, cancellationToken: cancellationToken).ConfigureAwait(false);
+        var (dsse, trust) = ProvenanceJsonParser.Parse(document.RootElement);
+
+        if (!trust.Verified)
+        {
+            return Problem(context, "Unverified provenance", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "trust.verified must be true.");
+        }
+
+        await eventLog.AttachStatementProvenanceAsync(statementId, dsse, trust, cancellationToken).ConfigureAwait(false);
+    }
+    catch (JsonException ex)
+    {
+        return Problem(context, "Invalid provenance payload", StatusCodes.Status400BadRequest, ProblemTypes.Validation, ex.Message);
+    }
+    catch (InvalidOperationException ex)
+    {
+        return Problem(context, "Statement not found", StatusCodes.Status404NotFound, ProblemTypes.NotFound, ex.Message);
+    }
+
+    return Results.Accepted($"/events/statements/{statementId}");
+});
+
+if (authorityConfigured)
+{
+    statementProvenanceEndpoint.RequireAuthorization(AdvisoryIngestPolicyName);
+}
+
 var loggingEnabled = concelierOptions.Telemetry?.EnableLogging ?? true;
 
 if (loggingEnabled)
@@ -1250,6 +1318,149 @@ IResult? EnsureTenantAuthorized(HttpContext context, string tenant)
     return null;
 }
 
+async Task<(Advisory Advisory, ImmutableArray<string> Aliases, string Fingerprint)?> ResolveAdvisoryAsync(
+    string advisoryKey,
+    IAdvisoryStore advisoryStore,
+    IAliasStore aliasStore,
+    CancellationToken cancellationToken)
+{
+    ArgumentNullException.ThrowIfNull(advisoryStore);
+    ArgumentNullException.ThrowIfNull(aliasStore);
+
+    var directCandidates = new List<string>();
+    if (!string.IsNullOrWhiteSpace(advisoryKey))
+    {
+        var trimmed = advisoryKey.Trim();
+        if (!string.IsNullOrWhiteSpace(trimmed))
+        {
+            directCandidates.Add(trimmed);
+            var upper = trimmed.ToUpperInvariant();
+            if (!string.Equals(upper, trimmed, StringComparison.Ordinal))
+            {
+                directCandidates.Add(upper);
+            }
+        }
+    }
+
+    foreach (var candidate in directCandidates.Distinct(StringComparer.OrdinalIgnoreCase))
+    {
+        var advisory = await advisoryStore.FindAsync(candidate, cancellationToken).ConfigureAwait(false);
+        if (advisory is not null)
+        {
+            return CreateResolution(advisory);
+        }
+    }
+
+    var aliasMatches = new List<AliasRecord>();
+    foreach (var (scheme, value) in BuildAliasLookups(advisoryKey))
+    {
+        var records = await aliasStore.GetByAliasAsync(scheme, value, cancellationToken).ConfigureAwait(false);
+        if (records.Count > 0)
+        {
+            aliasMatches.AddRange(records);
+        }
+    }
+
+    if (aliasMatches.Count == 0)
+    {
+        return null;
+    }
+
+    foreach (var candidate in aliasMatches
+                 .OrderByDescending(record => record.UpdatedAt)
+                 .ThenBy(record => record.AdvisoryKey, StringComparer.Ordinal)
+                 .Select(record => record.AdvisoryKey)
+                 .Distinct(StringComparer.OrdinalIgnoreCase))
+    {
+        var advisory = await advisoryStore.FindAsync(candidate, cancellationToken).ConfigureAwait(false);
+        if (advisory is not null)
+        {
+            return CreateResolution(advisory);
+        }
+    }
+
+    return null;
+}
+
+static (Advisory Advisory, ImmutableArray<string> Aliases, string Fingerprint) CreateResolution(Advisory advisory)
+{
+    var fingerprint = AdvisoryFingerprint.Compute(advisory);
+    var aliases = BuildAliasQuery(advisory);
+    return (advisory, aliases, fingerprint);
+}
+
+static ImmutableArray<string> BuildAliasQuery(Advisory advisory)
+{
+    var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+
+    if (!string.IsNullOrWhiteSpace(advisory.AdvisoryKey))
+    {
+        set.Add(advisory.AdvisoryKey.Trim());
+    }
+
+    foreach (var alias in advisory.Aliases)
+    {
+        if (!string.IsNullOrWhiteSpace(alias))
+        {
+            set.Add(alias.Trim());
+        }
+    }
+
+    if (set.Count == 0)
+    {
+        return ImmutableArray<string>.Empty;
+    }
+
+    var ordered = set
+        .OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
+        .ToList();
+
+    var canonical = advisory.AdvisoryKey?.Trim();
+    if (!string.IsNullOrWhiteSpace(canonical))
+    {
+        ordered.RemoveAll(value => string.Equals(value, canonical, StringComparison.OrdinalIgnoreCase));
+        ordered.Insert(0, canonical);
+    }
+
+    return ordered.ToImmutableArray();
+}
+
+static IReadOnlyList<(string Scheme, string Value)> BuildAliasLookups(string? candidate)
+{
+    var pairs = new List<(string Scheme, string Value)>();
+    var seen = new HashSet<string>(StringComparer.Ordinal);
+
+    void Add(string scheme, string? value)
+    {
+        if (string.IsNullOrWhiteSpace(scheme) || string.IsNullOrWhiteSpace(value))
+        {
+            return;
+        }
+
+        var trimmed = value.Trim();
+        if (trimmed.Length == 0)
+        {
+            return;
+        }
+
+        var key = $"{scheme}\u0001{trimmed}";
+        if (seen.Add(key))
+        {
+            pairs.Add((scheme, trimmed));
+        }
+    }
+
+    if (AliasSchemeRegistry.TryNormalize(candidate, out var normalized, out var scheme))
+    {
+        Add(scheme, normalized);
+    }
+
+    Add(AliasStoreConstants.UnscopedScheme, candidate);
+    Add(AliasStoreConstants.PrimaryScheme, candidate);
+
+    return pairs;
+}
+
 ImmutableHashSet<string> BuildFilterSet(StringValues values)
 {
     if (values.Count == 0)