Implement ledger metrics for observability and add tests for Ruby packages endpoints

- Added `LedgerMetrics` class to record write latency and total events for ledger operations.
- Created comprehensive tests for Ruby packages endpoints, covering scenarios for missing inventory, successful retrieval, and identifier handling.
- Introduced `TestSurfaceSecretsScope` for managing environment variables during tests.
- Developed `ProvenanceMongoExtensions` for attaching DSSE provenance and trust information to event documents.
- Implemented `EventProvenanceWriter` and `EventWriter` classes for managing event provenance in MongoDB.
- Established MongoDB indexes for efficient querying of events based on provenance and trust.
- Added models and JSON parsing logic for DSSE provenance and trust information.

src/Excititor/StellaOps.Excititor.WebService/Extensions/TelemetryExtensions.cs

@@ -7,6 +7,7 @@ using OpenTelemetry.Metrics;
 using OpenTelemetry.Resources;
 using OpenTelemetry.Trace;
 using StellaOps.Excititor.WebService.Options;
+using StellaOps.Excititor.WebService.Telemetry;
 using StellaOps.Ingestion.Telemetry;
 
 namespace StellaOps.Excititor.WebService.Extensions;
@@ -64,6 +65,7 @@ internal static class TelemetryExtensions
             {
                 metrics
                     .AddMeter(IngestionTelemetry.MeterName)
+                    .AddMeter(EvidenceTelemetry.MeterName)
                     .AddAspNetCoreInstrumentation()
                     .AddHttpClientInstrumentation()
                     .AddRuntimeInstrumentation();

src/Excititor/StellaOps.Excititor.WebService/Program.cs

@@ -29,6 +29,7 @@ using StellaOps.Excititor.WebService.Options;
 using StellaOps.Excititor.WebService.Services;
 using StellaOps.Excititor.Core.Aoc;
 using StellaOps.Excititor.WebService.Contracts;
+using StellaOps.Excititor.WebService.Telemetry;
 using MongoDB.Driver;
 using MongoDB.Bson;
 
@@ -216,6 +217,7 @@ app.MapPost("/ingest/vex", async (
     }
     catch (ExcititorAocGuardException guardException)
     {
+        EvidenceTelemetry.RecordGuardViolations(tenant, "ingest", guardException);
         logger.LogWarning(
             guardException,
             "AOC guard rejected VEX ingest tenant={Tenant} digest={Digest}",
@@ -478,8 +480,27 @@ app.MapGet("/v1/vex/observations/{vulnerabilityId}/{productKey}", async (
         since,
         limit);
 
-    var result = await projectionService.QueryAsync(request, cancellationToken).ConfigureAwait(false);
-    var statements = result.Statements
+    VexObservationProjectionResult result;
+    try
+    {
+        result = await projectionService.QueryAsync(request, cancellationToken).ConfigureAwait(false);
+    }
+    catch (OperationCanceledException)
+    {
+        EvidenceTelemetry.RecordObservationOutcome(tenant, "cancelled");
+        throw;
+    }
+    catch
+    {
+        EvidenceTelemetry.RecordObservationOutcome(tenant, "error");
+        throw;
+    }
+
+    var projectionStatements = result.Statements;
+    EvidenceTelemetry.RecordObservationOutcome(tenant, "success", projectionStatements.Count, result.Truncated);
+    EvidenceTelemetry.RecordSignatureStatus(tenant, projectionStatements);
+
+    var statements = projectionStatements
         .Select(ToResponse)
         .ToList();
 
@@ -575,6 +596,7 @@ app.MapPost("/aoc/verify", async (
         }
         catch (ExcititorAocGuardException guardException)
         {
+            EvidenceTelemetry.RecordGuardViolations(tenant, "aoc_verify", guardException);
             checkedCount++;
             foreach (var violation in guardException.Violations)
             {

src/Excititor/StellaOps.Excititor.WebService/Services/VexObservationProjectionService.cs

@@ -6,6 +6,7 @@ using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using StellaOps.Excititor.Core;
+using StellaOps.Excititor.Storage.Mongo;
 
 namespace StellaOps.Excititor.WebService.Services;
 

src/Excititor/StellaOps.Excititor.WebService/Telemetry/EvidenceTelemetry.cs

@@ -0,0 +1,154 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.Metrics;
+using StellaOps.Excititor.Core.Aoc;
+using StellaOps.Excititor.WebService.Services;
+
+namespace StellaOps.Excititor.WebService.Telemetry;
+
+internal static class EvidenceTelemetry
+{
+    public const string MeterName = "StellaOps.Excititor.WebService.Evidence";
+
+    private static readonly Meter Meter = new(MeterName);
+
+    private static readonly Counter<long> ObservationRequestCounter =
+        Meter.CreateCounter<long>(
+            "excititor.vex.observation.requests",
+            unit: "requests",
+            description: "Number of observation projection requests handled by the evidence APIs.");
+
+    private static readonly Histogram<int> ObservationStatementHistogram =
+        Meter.CreateHistogram<int>(
+            "excititor.vex.observation.statement_count",
+            unit: "statements",
+            description: "Distribution of statements returned per observation projection request.");
+
+    private static readonly Counter<long> SignatureStatusCounter =
+        Meter.CreateCounter<long>(
+            "excititor.vex.signature.status",
+            unit: "statements",
+            description: "Signature verification status counts for observation statements.");
+
+    private static readonly Counter<long> GuardViolationCounter =
+        Meter.CreateCounter<long>(
+            "excititor.vex.aoc.guard_violations",
+            unit: "violations",
+            description: "Aggregated count of AOC guard violations detected by Excititor evidence APIs.");
+
+    public static void RecordObservationOutcome(string? tenant, string outcome, int returnedCount = 0, bool truncated = false)
+    {
+        var normalizedTenant = NormalizeTenant(tenant);
+        var tags = new[]
+        {
+            new KeyValuePair<string, object?>("tenant", normalizedTenant),
+            new KeyValuePair<string, object?>("outcome", outcome),
+            new KeyValuePair<string, object?>("truncated", truncated),
+        };
+
+        ObservationRequestCounter.Add(1, tags);
+
+        if (!string.Equals(outcome, "success", StringComparison.OrdinalIgnoreCase))
+        {
+            return;
+        }
+
+        ObservationStatementHistogram.Record(
+            returnedCount,
+            new[]
+            {
+                new KeyValuePair<string, object?>("tenant", normalizedTenant),
+                new KeyValuePair<string, object?>("outcome", outcome),
+            });
+    }
+
+    public static void RecordSignatureStatus(string? tenant, IReadOnlyList<VexObservationStatementProjection> statements)
+    {
+        if (statements is null || statements.Count == 0)
+        {
+            return;
+        }
+
+        var normalizedTenant = NormalizeTenant(tenant);
+        var missing = 0;
+        var unverified = 0;
+
+        foreach (var statement in statements)
+        {
+            var signature = statement.Signature;
+            if (signature is null)
+            {
+                missing++;
+                continue;
+            }
+
+            if (signature.VerifiedAt is null)
+            {
+                unverified++;
+            }
+        }
+
+        if (missing > 0)
+        {
+            SignatureStatusCounter.Add(
+                missing,
+                new[]
+                {
+                    new KeyValuePair<string, object?>("tenant", normalizedTenant),
+                    new KeyValuePair<string, object?>("status", "missing"),
+                });
+        }
+
+        if (unverified > 0)
+        {
+            SignatureStatusCounter.Add(
+                unverified,
+                new[]
+                {
+                    new KeyValuePair<string, object?>("tenant", normalizedTenant),
+                    new KeyValuePair<string, object?>("status", "unverified"),
+                });
+        }
+    }
+
+    public static void RecordGuardViolations(string? tenant, string surface, ExcititorAocGuardException exception)
+    {
+        var normalizedTenant = NormalizeTenant(tenant);
+        var normalizedSurface = NormalizeSurface(surface);
+
+        if (exception.Violations.IsDefaultOrEmpty)
+        {
+            GuardViolationCounter.Add(
+                1,
+                new[]
+                {
+                    new KeyValuePair<string, object?>("tenant", normalizedTenant),
+                    new KeyValuePair<string, object?>("surface", normalizedSurface),
+                    new KeyValuePair<string, object?>("code", exception.PrimaryErrorCode),
+                });
+            return;
+        }
+
+        foreach (var violation in exception.Violations)
+        {
+            var code = string.IsNullOrWhiteSpace(violation.ErrorCode)
+                ? exception.PrimaryErrorCode
+                : violation.ErrorCode;
+
+            GuardViolationCounter.Add(
+                1,
+                new[]
+                {
+                    new KeyValuePair<string, object?>("tenant", normalizedTenant),
+                    new KeyValuePair<string, object?>("surface", normalizedSurface),
+                    new KeyValuePair<string, object?>("code", code),
+                });
+        }
+    }
+
+    private static string NormalizeTenant(string? tenant)
+        => string.IsNullOrWhiteSpace(tenant) ? "default" : tenant;
+
+    private static string NormalizeSurface(string? surface)
+        => string.IsNullOrWhiteSpace(surface) ? "unknown" : surface.ToLowerInvariant();
+}