Implement ledger metrics for observability and add tests for Ruby packages endpoints

- Added `LedgerMetrics` class to record write latency and total events for ledger operations.
- Created comprehensive tests for Ruby packages endpoints, covering scenarios for missing inventory, successful retrieval, and identifier handling.
- Introduced `TestSurfaceSecretsScope` for managing environment variables during tests.
- Developed `ProvenanceMongoExtensions` for attaching DSSE provenance and trust information to event documents.
- Implemented `EventProvenanceWriter` and `EventWriter` classes for managing event provenance in MongoDB.
- Established MongoDB indexes for efficient querying of events based on provenance and trust.
- Added models and JSON parsing logic for DSSE provenance and trust information.

src/Excititor/StellaOps.Excititor.WebService/Program.cs

@@ -29,6 +29,7 @@ using StellaOps.Excititor.WebService.Options;
 using StellaOps.Excititor.WebService.Services;
 using StellaOps.Excititor.Core.Aoc;
 using StellaOps.Excititor.WebService.Contracts;
+using StellaOps.Excititor.WebService.Telemetry;
 using MongoDB.Driver;
 using MongoDB.Bson;
 
@@ -216,6 +217,7 @@ app.MapPost("/ingest/vex", async (
     }
     catch (ExcititorAocGuardException guardException)
     {
+        EvidenceTelemetry.RecordGuardViolations(tenant, "ingest", guardException);
         logger.LogWarning(
             guardException,
             "AOC guard rejected VEX ingest tenant={Tenant} digest={Digest}",
@@ -478,8 +480,27 @@ app.MapGet("/v1/vex/observations/{vulnerabilityId}/{productKey}", async (
         since,
         limit);
 
-    var result = await projectionService.QueryAsync(request, cancellationToken).ConfigureAwait(false);
-    var statements = result.Statements
+    VexObservationProjectionResult result;
+    try
+    {
+        result = await projectionService.QueryAsync(request, cancellationToken).ConfigureAwait(false);
+    }
+    catch (OperationCanceledException)
+    {
+        EvidenceTelemetry.RecordObservationOutcome(tenant, "cancelled");
+        throw;
+    }
+    catch
+    {
+        EvidenceTelemetry.RecordObservationOutcome(tenant, "error");
+        throw;
+    }
+
+    var projectionStatements = result.Statements;
+    EvidenceTelemetry.RecordObservationOutcome(tenant, "success", projectionStatements.Count, result.Truncated);
+    EvidenceTelemetry.RecordSignatureStatus(tenant, projectionStatements);
+
+    var statements = projectionStatements
         .Select(ToResponse)
         .ToList();
 
@@ -575,6 +596,7 @@ app.MapPost("/aoc/verify", async (
         }
         catch (ExcititorAocGuardException guardException)
         {
+            EvidenceTelemetry.RecordGuardViolations(tenant, "aoc_verify", guardException);
             checkedCount++;
             foreach (var violation in guardException.Violations)
             {