Implement ledger metrics for observability and add tests for Ruby packages endpoints

- Added `LedgerMetrics` class to record write latency and total events for ledger operations.
- Created comprehensive tests for Ruby packages endpoints, covering scenarios for missing inventory, successful retrieval, and identifier handling.
- Introduced `TestSurfaceSecretsScope` for managing environment variables during tests.
- Developed `ProvenanceMongoExtensions` for attaching DSSE provenance and trust information to event documents.
- Implemented `EventProvenanceWriter` and `EventWriter` classes for managing event provenance in MongoDB.
- Established MongoDB indexes for efficient querying of events based on provenance and trust.
- Added models and JSON parsing logic for DSSE provenance and trust information.

src/__Libraries/StellaOps.Provenance.Mongo/DsseProvenanceModels.cs

@@ -0,0 +1,43 @@
+using System.Collections.Generic;
+using MongoDB.Bson;
+
+namespace StellaOps.Provenance.Mongo;
+
+public sealed class DsseKeyInfo
+{
+    public string KeyId { get; set; } = default!;          // e.g. "cosign:SHA256-PKIX:..."
+    public string? Issuer { get; set; }                    // e.g. Fulcio issuer, KMS URI, X.509 CN
+    public string? Algo { get; set; }                      // "ECDSA" | "RSA" | "Ed25519" | "Dilithium"
+}
+
+public sealed class DsseRekorInfo
+{
+    public long LogIndex { get; set; }                     // Rekor log index
+    public string Uuid { get; set; } = default!;           // Rekor entry UUID
+    public long? IntegratedTime { get; set; }              // unix timestamp (seconds)
+    public long? MirrorSeq { get; set; }                   // optional mirror sequence in Proof-Market ledger
+}
+
+public sealed class DsseChainLink
+{
+    public string Type { get; set; } = default!;           // e.g. "build" | "sbom" | "scan"
+    public string Id { get; set; } = default!;             // e.g. "att:build#..."
+    public string Digest { get; set; } = default!;         // sha256 of DSSE envelope or payload
+}
+
+public sealed class DsseProvenance
+{
+    public string EnvelopeDigest { get; set; } = default!; // sha256 of envelope (not payload)
+    public string PayloadType { get; set; } = default!;    // "application/vnd.in-toto+json"
+    public DsseKeyInfo Key { get; set; } = new();
+    public DsseRekorInfo? Rekor { get; set; }
+    public IReadOnlyCollection<DsseChainLink>? Chain { get; set; }
+}
+
+public sealed class TrustInfo
+{
+    public bool Verified { get; set; }                     // local cryptographic verification
+    public string? Verifier { get; set; }                  // e.g. "Authority@stella"
+    public int? Witnesses { get; set; }                    // number of verified transparency witnesses
+    public double? PolicyScore { get; set; }               // lattice / policy score (0..1)
+}

src/__Libraries/StellaOps.Provenance.Mongo/ProvenanceJsonParser.cs

@@ -0,0 +1,203 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace StellaOps.Provenance.Mongo;
+
+public static class ProvenanceJsonParser
+{
+    public static (DsseProvenance Dsse, TrustInfo Trust) Parse(JsonElement root, TrustInfo? trustOverride = null)
+    {
+        var dsse = ParseDsse(root);
+        var trust = trustOverride ?? ParseTrust(root) ?? throw new InvalidOperationException("Provenance metadata missing trust block.");
+        return (dsse, trust);
+    }
+
+    public static (DsseProvenance Dsse, TrustInfo Trust) Parse(string json, TrustInfo? trustOverride = null)
+    {
+        using var document = JsonDocument.Parse(json);
+        return Parse(document.RootElement, trustOverride);
+    }
+
+    public static async Task<(DsseProvenance Dsse, TrustInfo Trust)> ParseAsync(
+        Stream utf8JsonStream,
+        TrustInfo? trustOverride = null,
+        CancellationToken cancellationToken = default)
+    {
+        var document = await JsonDocument.ParseAsync(utf8JsonStream, cancellationToken: cancellationToken).ConfigureAwait(false);
+        using (document)
+        {
+            return Parse(document.RootElement, trustOverride);
+        }
+    }
+
+    private static DsseProvenance ParseDsse(JsonElement root)
+    {
+        if (!root.TryGetProperty("dsse", out var dsseElement) || dsseElement.ValueKind != JsonValueKind.Object)
+        {
+            throw new InvalidOperationException("Provenance metadata missing dsse block.");
+        }
+
+        var keyElement = GetRequiredProperty(dsseElement, "key");
+        var dsse = new DsseProvenance
+        {
+            EnvelopeDigest = GetRequiredString(dsseElement, "envelopeDigest"),
+            PayloadType = GetRequiredString(dsseElement, "payloadType"),
+            Key = new DsseKeyInfo
+            {
+                KeyId = GetRequiredString(keyElement, "keyId"),
+                Issuer = GetOptionalString(keyElement, "issuer"),
+                Algo = GetOptionalString(keyElement, "algo"),
+            },
+            Chain = ParseChain(dsseElement)
+        };
+
+        if (dsseElement.TryGetProperty("rekor", out var rekorElement) && rekorElement.ValueKind == JsonValueKind.Object)
+        {
+            dsse.Rekor = new DsseRekorInfo
+            {
+                LogIndex = GetInt64(rekorElement, "logIndex"),
+                Uuid = GetRequiredString(rekorElement, "uuid"),
+                IntegratedTime = GetOptionalInt64(rekorElement, "integratedTime"),
+                MirrorSeq = GetOptionalInt64(rekorElement, "mirrorSeq")
+            };
+        }
+
+        return dsse;
+    }
+
+    private static IReadOnlyCollection<DsseChainLink>? ParseChain(JsonElement dsseElement)
+    {
+        if (!dsseElement.TryGetProperty("chain", out var chainElement) || chainElement.ValueKind != JsonValueKind.Array || chainElement.GetArrayLength() == 0)
+        {
+            return null;
+        }
+
+        var links = new List<DsseChainLink>(chainElement.GetArrayLength());
+        foreach (var entry in chainElement.EnumerateArray())
+        {
+            if (entry.ValueKind != JsonValueKind.Object)
+            {
+                continue;
+            }
+
+            var type = GetOptionalString(entry, "type");
+            var id = GetOptionalString(entry, "id");
+            var digest = GetOptionalString(entry, "digest");
+
+            if (string.IsNullOrEmpty(type) || string.IsNullOrEmpty(id) || string.IsNullOrEmpty(digest))
+            {
+                continue;
+            }
+
+            links.Add(new DsseChainLink
+            {
+                Type = type,
+                Id = id,
+                Digest = digest
+            });
+        }
+
+        return links.Count == 0 ? null : links;
+    }
+
+    private static TrustInfo? ParseTrust(JsonElement root)
+    {
+        if (!root.TryGetProperty("trust", out var trustElement) || trustElement.ValueKind != JsonValueKind.Object)
+        {
+            return null;
+        }
+
+        var trust = new TrustInfo
+        {
+            Verified = trustElement.TryGetProperty("verified", out var verified) && verified.ValueKind == JsonValueKind.True,
+            Verifier = GetOptionalString(trustElement, "verifier"),
+            Witnesses = trustElement.TryGetProperty("witnesses", out var witnessesElement) && witnessesElement.TryGetInt32(out var witnesses)
+                ? witnesses
+                : null,
+            PolicyScore = trustElement.TryGetProperty("policyScore", out var scoreElement) && scoreElement.TryGetDouble(out var score)
+                ? score
+                : null
+        };
+
+        return trust;
+    }
+
+    private static JsonElement GetRequiredProperty(JsonElement parent, string name)
+    {
+        if (!parent.TryGetProperty(name, out var property) || property.ValueKind == JsonValueKind.Null)
+        {
+            throw new InvalidOperationException($"Provenance metadata missing required property {name}.");
+        }
+
+        return property;
+    }
+
+    private static string GetRequiredString(JsonElement parent, string name)
+    {
+        var element = GetRequiredProperty(parent, name);
+        if (element.ValueKind is JsonValueKind.String)
+        {
+            var value = element.GetString();
+            if (!string.IsNullOrWhiteSpace(value))
+            {
+                return value;
+            }
+        }
+
+        throw new InvalidOperationException($"Provenance metadata property {name} must be a non-empty string.");
+    }
+
+    private static string? GetOptionalString(JsonElement parent, string name)
+    {
+        if (!parent.TryGetProperty(name, out var element))
+        {
+            return null;
+        }
+
+        return element.ValueKind == JsonValueKind.String ? element.GetString() : null;
+    }
+
+    private static long GetInt64(JsonElement parent, string name)
+    {
+        if (!parent.TryGetProperty(name, out var element))
+        {
+            throw new InvalidOperationException($"Provenance metadata missing {name}.");
+        }
+
+        if (element.TryGetInt64(out var value))
+        {
+            return value;
+        }
+
+        if (element.ValueKind == JsonValueKind.String && long.TryParse(element.GetString(), out value))
+        {
+            return value;
+        }
+
+        throw new InvalidOperationException($"Provenance metadata property {name} must be an integer.");
+    }
+
+    private static long? GetOptionalInt64(JsonElement parent, string name)
+    {
+        if (!parent.TryGetProperty(name, out var element))
+        {
+            return null;
+        }
+
+        if (element.TryGetInt64(out var value))
+        {
+            return value;
+        }
+
+        if (element.ValueKind == JsonValueKind.String && long.TryParse(element.GetString(), out value))
+        {
+            return value;
+        }
+
+        return null;
+    }
+}

src/__Libraries/StellaOps.Provenance.Mongo/ProvenanceMongoExtensions.cs

@@ -0,0 +1,142 @@
+using MongoDB.Bson;
+
+namespace StellaOps.Provenance.Mongo;
+
+public static class ProvenanceMongoExtensions
+{
+    private const string ProvenanceFieldName = "provenance";
+    private const string DsseFieldName = "dsse";
+    private const string TrustFieldName = "trust";
+    private const string ChainFieldName = "chain";
+    private static BsonValue StringOrNull(string? value) =>
+        value is null ? BsonNull.Value : new BsonString(value);
+
+    /// <summary>
+    /// Attach DSSE provenance + trust info to an event document in-place.
+    /// Designed for generic BsonDocument-based event envelopes.
+    /// </summary>
+    public static BsonDocument AttachDsseProvenance(
+        this BsonDocument eventDoc,
+        DsseProvenance dsse,
+        TrustInfo trust)
+    {
+        if (eventDoc is null) throw new ArgumentNullException(nameof(eventDoc));
+        if (dsse is null) throw new ArgumentNullException(nameof(dsse));
+        if (trust is null) throw new ArgumentNullException(nameof(trust));
+
+        var dsseDoc = new BsonDocument
+        {
+            { "envelopeDigest", dsse.EnvelopeDigest },
+            { "payloadType", dsse.PayloadType },
+                    { "key", new BsonDocument
+                        {
+                            { "keyId", dsse.Key.KeyId },
+                            { "issuer", StringOrNull(dsse.Key.Issuer) },
+                            { "algo", StringOrNull(dsse.Key.Algo) }
+                        }
+                    }
+        };
+
+        if (dsse.Rekor is not null)
+        {
+            var rekorDoc = new BsonDocument
+            {
+                { "logIndex", dsse.Rekor.LogIndex },
+                { "uuid", dsse.Rekor.Uuid }
+            };
+
+            if (dsse.Rekor.IntegratedTime is not null)
+                rekorDoc.Add("integratedTime", dsse.Rekor.IntegratedTime);
+
+            if (dsse.Rekor.MirrorSeq is not null)
+                rekorDoc.Add("mirrorSeq", dsse.Rekor.MirrorSeq);
+
+            dsseDoc.Add("rekor", rekorDoc);
+        }
+
+        if (dsse.Chain is not null && dsse.Chain.Count > 0)
+        {
+            var chainArray = new BsonArray();
+            foreach (var link in dsse.Chain)
+            {
+                chainArray.Add(new BsonDocument
+                {
+                    { "type", link.Type },
+                    { "id", link.Id },
+                    { "digest", link.Digest }
+                });
+            }
+
+            dsseDoc.Add(ChainFieldName, chainArray);
+        }
+
+        var trustDoc = new BsonDocument
+        {
+            { "verified", trust.Verified },
+            { "verifier", StringOrNull(trust.Verifier) }
+        };
+
+        if (trust.Witnesses is not null)
+            trustDoc.Add("witnesses", trust.Witnesses);
+
+        if (trust.PolicyScore is not null)
+            trustDoc.Add("policyScore", trust.PolicyScore);
+
+        var provenanceDoc = new BsonDocument
+        {
+            { DsseFieldName, dsseDoc }
+        };
+
+        eventDoc[ProvenanceFieldName] = provenanceDoc;
+        eventDoc[TrustFieldName] = trustDoc;
+
+        return eventDoc;
+    }
+
+    /// <summary>
+    /// Helper to query for "cryptographically proven" events:
+    /// kind + subject.digest.sha256 + presence of Rekor logIndex + trust.verified = true.
+    /// </summary>
+    public static BsonDocument BuildProvenVexFilter(
+        string kind,
+        string subjectDigestSha256)
+    {
+        return new BsonDocument
+        {
+            { "kind", kind },
+            { "subject.digest.sha256", subjectDigestSha256 },
+            { $"{ProvenanceFieldName}.{DsseFieldName}.rekor.logIndex", new BsonDocument("$exists", true) },
+            { $"{TrustFieldName}.verified", true }
+        };
+    }
+
+    /// <summary>
+    /// Helper to query for events influencing policy without solid provenance.
+    /// </summary>
+    public static BsonDocument BuildUnprovenEvidenceFilter(
+        IEnumerable<string> kinds)
+    {
+        var kindsArray = new BsonArray(kinds);
+
+        return new BsonDocument
+        {
+            {
+                "kind", new BsonDocument("$in", kindsArray)
+            },
+            {
+                "$or", new BsonArray
+                {
+                    new BsonDocument
+                    {
+                        { $"{TrustFieldName}.verified", new BsonDocument("$ne", true) }
+                    },
+                    new BsonDocument
+                    {
+                        { $"{ProvenanceFieldName}.{DsseFieldName}.rekor.logIndex",
+                          new BsonDocument("$exists", false) }
+                    }
+                }
+            }
+        };
+    }
+}

src/__Libraries/StellaOps.Provenance.Mongo/StellaOps.Provenance.Mongo.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="MongoDB.Driver" Version="3.5.0" />
+  </ItemGroup>
+
+</Project>