using System.Collections.Concurrent; using System.Globalization; using System.Text; namespace StellaOps.Scanner.WebService.Services; internal sealed class OfflineKitMetricsStore { private static readonly double[] DefaultLatencyBucketsSeconds = { 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10 }; private readonly ConcurrentDictionary _imports = new(); private readonly ConcurrentDictionary _attestationVerifyLatency = new(); private readonly ConcurrentDictionary _rekorInclusionLatency = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _rekorSuccess = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _rekorRetry = new(StringComparer.Ordinal); public void RecordImport(string status, string tenantId) { status = NormalizeLabelValue(status, "unknown"); tenantId = NormalizeLabelValue(tenantId, "unknown"); _imports.AddOrUpdate(new ImportCounterKey(tenantId, status), 1, static (_, current) => current + 1); } public void RecordAttestationVerifyLatency(string attestationType, double seconds, bool success) { attestationType = NormalizeLabelValue(attestationType, "unknown"); seconds = ClampSeconds(seconds); var key = new TwoLabelKey(attestationType, success ? "true" : "false"); var histogram = _attestationVerifyLatency.GetOrAdd(key, _ => new Histogram(DefaultLatencyBucketsSeconds)); histogram.Record(seconds); } public void RecordRekorSuccess(string mode) { mode = NormalizeLabelValue(mode, "unknown"); _rekorSuccess.AddOrUpdate(mode, 1, static (_, current) => current + 1); } public void RecordRekorRetry(string reason) { reason = NormalizeLabelValue(reason, "unknown"); _rekorRetry.AddOrUpdate(reason, 1, static (_, current) => current + 1); } public void RecordRekorInclusionLatency(double seconds, bool success) { seconds = ClampSeconds(seconds); var key = success ? "true" : "false"; var histogram = _rekorInclusionLatency.GetOrAdd(key, _ => new Histogram(DefaultLatencyBucketsSeconds)); histogram.Record(seconds); } public string RenderPrometheus() { var builder = new StringBuilder(capacity: 4096); AppendCounterHeader(builder, "offlinekit_import_total", "Total number of offline kit import attempts"); foreach (var (key, value) in _imports.OrderBy(kv => kv.Key.TenantId, StringComparer.Ordinal) .ThenBy(kv => kv.Key.Status, StringComparer.Ordinal)) { builder.Append("offlinekit_import_total{tenant_id=\""); builder.Append(EscapeLabelValue(key.TenantId)); builder.Append("\",status=\""); builder.Append(EscapeLabelValue(key.Status)); builder.Append("\"} "); builder.Append(value.ToString(CultureInfo.InvariantCulture)); builder.Append('\n'); } AppendHistogramTwoLabels( builder, name: "offlinekit_attestation_verify_latency_seconds", help: "Time taken to verify attestations during import", labelA: "attestation_type", labelB: "success", histograms: _attestationVerifyLatency); AppendCounterHeader(builder, "attestor_rekor_success_total", "Successful Rekor verification count"); foreach (var (key, value) in _rekorSuccess.OrderBy(kv => kv.Key, StringComparer.Ordinal)) { builder.Append("attestor_rekor_success_total{mode=\""); builder.Append(EscapeLabelValue(key)); builder.Append("\"} "); builder.Append(value.ToString(CultureInfo.InvariantCulture)); builder.Append('\n'); } AppendCounterHeader(builder, "attestor_rekor_retry_total", "Rekor verification retry count"); foreach (var (key, value) in _rekorRetry.OrderBy(kv => kv.Key, StringComparer.Ordinal)) { builder.Append("attestor_rekor_retry_total{reason=\""); builder.Append(EscapeLabelValue(key)); builder.Append("\"} "); builder.Append(value.ToString(CultureInfo.InvariantCulture)); builder.Append('\n'); } AppendHistogramOneLabel( builder, name: "rekor_inclusion_latency", help: "Time to verify Rekor inclusion proof", label: "success", histograms: _rekorInclusionLatency); return builder.ToString(); } private static void AppendCounterHeader(StringBuilder builder, string name, string help) { builder.Append("# HELP "); builder.Append(name); builder.Append(' '); builder.Append(help); builder.Append('\n'); builder.Append("# TYPE "); builder.Append(name); builder.Append(" counter\n"); } private static void AppendHistogramTwoLabels( StringBuilder builder, string name, string help, string labelA, string labelB, ConcurrentDictionary histograms) { builder.Append("# HELP "); builder.Append(name); builder.Append(' '); builder.Append(help); builder.Append('\n'); builder.Append("# TYPE "); builder.Append(name); builder.Append(" histogram\n"); foreach (var grouping in histograms.OrderBy(kv => kv.Key.LabelA, StringComparer.Ordinal) .ThenBy(kv => kv.Key.LabelB, StringComparer.Ordinal)) { var labels = $"{labelA}=\"{EscapeLabelValue(grouping.Key.LabelA)}\",{labelB}=\"{EscapeLabelValue(grouping.Key.LabelB)}\""; AppendHistogramSeries(builder, name, labels, grouping.Value.Snapshot()); } } private static void AppendHistogramOneLabel( StringBuilder builder, string name, string help, string label, ConcurrentDictionary histograms) { builder.Append("# HELP "); builder.Append(name); builder.Append(' '); builder.Append(help); builder.Append('\n'); builder.Append("# TYPE "); builder.Append(name); builder.Append(" histogram\n"); foreach (var grouping in histograms.OrderBy(kv => kv.Key, StringComparer.Ordinal)) { var labels = $"{label}=\"{EscapeLabelValue(grouping.Key)}\""; AppendHistogramSeries(builder, name, labels, grouping.Value.Snapshot()); } } private static void AppendHistogramSeries( StringBuilder builder, string name, string labels, HistogramSnapshot snapshot) { long cumulative = 0; for (var i = 0; i < snapshot.BucketUpperBounds.Length; i++) { cumulative += snapshot.BucketCounts[i]; builder.Append(name); builder.Append("_bucket{"); builder.Append(labels); builder.Append(",le=\""); builder.Append(snapshot.BucketUpperBounds[i].ToString("G", CultureInfo.InvariantCulture)); builder.Append("\"} "); builder.Append(cumulative.ToString(CultureInfo.InvariantCulture)); builder.Append('\n'); } cumulative += snapshot.BucketCounts[^1]; builder.Append(name); builder.Append("_bucket{"); builder.Append(labels); builder.Append(",le=\"+Inf\"} "); builder.Append(cumulative.ToString(CultureInfo.InvariantCulture)); builder.Append('\n'); builder.Append(name); builder.Append("_sum{"); builder.Append(labels); builder.Append("} "); builder.Append(snapshot.SumSeconds.ToString("G", CultureInfo.InvariantCulture)); builder.Append('\n'); builder.Append(name); builder.Append("_count{"); builder.Append(labels); builder.Append("} "); builder.Append(snapshot.Count.ToString(CultureInfo.InvariantCulture)); builder.Append('\n'); } private static double ClampSeconds(double seconds) => double.IsNaN(seconds) || double.IsInfinity(seconds) || seconds < 0 ? 0 : seconds; private static string NormalizeLabelValue(string? value, string fallback) => string.IsNullOrWhiteSpace(value) ? fallback : value.Trim(); private static string EscapeLabelValue(string value) => value.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal); private sealed class Histogram { private readonly double[] _bucketUpperBounds; private readonly long[] _bucketCounts; private long _count; private double _sumSeconds; private readonly object _lock = new(); public Histogram(double[] bucketUpperBounds) { _bucketUpperBounds = bucketUpperBounds ?? throw new ArgumentNullException(nameof(bucketUpperBounds)); _bucketCounts = new long[_bucketUpperBounds.Length + 1]; } public void Record(double seconds) { lock (_lock) { _count++; _sumSeconds += seconds; var bucketIndex = _bucketUpperBounds.Length; for (var i = 0; i < _bucketUpperBounds.Length; i++) { if (seconds <= _bucketUpperBounds[i]) { bucketIndex = i; break; } } _bucketCounts[bucketIndex]++; } } public HistogramSnapshot Snapshot() { lock (_lock) { return new HistogramSnapshot( (double[])_bucketUpperBounds.Clone(), (long[])_bucketCounts.Clone(), _count, _sumSeconds); } } } private sealed record HistogramSnapshot( double[] BucketUpperBounds, long[] BucketCounts, long Count, double SumSeconds); private sealed record ImportCounterKey(string TenantId, string Status); private sealed record TwoLabelKey(string LabelA, string LabelB); }