295 lines
10 KiB
C#
295 lines
10 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Globalization;
|
|
using System.Text;
|
|
|
|
namespace StellaOps.Scanner.WebService.Services;
|
|
|
|
internal sealed class OfflineKitMetricsStore
|
|
{
|
|
private static readonly double[] DefaultLatencyBucketsSeconds =
|
|
{
|
|
0.001,
|
|
0.0025,
|
|
0.005,
|
|
0.01,
|
|
0.025,
|
|
0.05,
|
|
0.1,
|
|
0.25,
|
|
0.5,
|
|
1,
|
|
2.5,
|
|
5,
|
|
10
|
|
};
|
|
|
|
private readonly ConcurrentDictionary<ImportCounterKey, long> _imports = new();
|
|
private readonly ConcurrentDictionary<TwoLabelKey, Histogram> _attestationVerifyLatency = new();
|
|
private readonly ConcurrentDictionary<string, Histogram> _rekorInclusionLatency = new(StringComparer.Ordinal);
|
|
private readonly ConcurrentDictionary<string, long> _rekorSuccess = new(StringComparer.Ordinal);
|
|
private readonly ConcurrentDictionary<string, long> _rekorRetry = new(StringComparer.Ordinal);
|
|
|
|
public void RecordImport(string status, string tenantId)
|
|
{
|
|
status = NormalizeLabelValue(status, "unknown");
|
|
tenantId = NormalizeLabelValue(tenantId, "unknown");
|
|
_imports.AddOrUpdate(new ImportCounterKey(tenantId, status), 1, static (_, current) => current + 1);
|
|
}
|
|
|
|
public void RecordAttestationVerifyLatency(string attestationType, double seconds, bool success)
|
|
{
|
|
attestationType = NormalizeLabelValue(attestationType, "unknown");
|
|
seconds = ClampSeconds(seconds);
|
|
var key = new TwoLabelKey(attestationType, success ? "true" : "false");
|
|
var histogram = _attestationVerifyLatency.GetOrAdd(key, _ => new Histogram(DefaultLatencyBucketsSeconds));
|
|
histogram.Record(seconds);
|
|
}
|
|
|
|
public void RecordRekorSuccess(string mode)
|
|
{
|
|
mode = NormalizeLabelValue(mode, "unknown");
|
|
_rekorSuccess.AddOrUpdate(mode, 1, static (_, current) => current + 1);
|
|
}
|
|
|
|
public void RecordRekorRetry(string reason)
|
|
{
|
|
reason = NormalizeLabelValue(reason, "unknown");
|
|
_rekorRetry.AddOrUpdate(reason, 1, static (_, current) => current + 1);
|
|
}
|
|
|
|
public void RecordRekorInclusionLatency(double seconds, bool success)
|
|
{
|
|
seconds = ClampSeconds(seconds);
|
|
var key = success ? "true" : "false";
|
|
var histogram = _rekorInclusionLatency.GetOrAdd(key, _ => new Histogram(DefaultLatencyBucketsSeconds));
|
|
histogram.Record(seconds);
|
|
}
|
|
|
|
public string RenderPrometheus()
|
|
{
|
|
var builder = new StringBuilder(capacity: 4096);
|
|
|
|
AppendCounterHeader(builder, "offlinekit_import_total", "Total number of offline kit import attempts");
|
|
foreach (var (key, value) in _imports.OrderBy(kv => kv.Key.TenantId, StringComparer.Ordinal)
|
|
.ThenBy(kv => kv.Key.Status, StringComparer.Ordinal))
|
|
{
|
|
builder.Append("offlinekit_import_total{tenant_id=\"");
|
|
builder.Append(EscapeLabelValue(key.TenantId));
|
|
builder.Append("\",status=\"");
|
|
builder.Append(EscapeLabelValue(key.Status));
|
|
builder.Append("\"} ");
|
|
builder.Append(value.ToString(CultureInfo.InvariantCulture));
|
|
builder.Append('\n');
|
|
}
|
|
|
|
AppendHistogramTwoLabels(
|
|
builder,
|
|
name: "offlinekit_attestation_verify_latency_seconds",
|
|
help: "Time taken to verify attestations during import",
|
|
labelA: "attestation_type",
|
|
labelB: "success",
|
|
histograms: _attestationVerifyLatency);
|
|
|
|
AppendCounterHeader(builder, "attestor_rekor_success_total", "Successful Rekor verification count");
|
|
foreach (var (key, value) in _rekorSuccess.OrderBy(kv => kv.Key, StringComparer.Ordinal))
|
|
{
|
|
builder.Append("attestor_rekor_success_total{mode=\"");
|
|
builder.Append(EscapeLabelValue(key));
|
|
builder.Append("\"} ");
|
|
builder.Append(value.ToString(CultureInfo.InvariantCulture));
|
|
builder.Append('\n');
|
|
}
|
|
|
|
AppendCounterHeader(builder, "attestor_rekor_retry_total", "Rekor verification retry count");
|
|
foreach (var (key, value) in _rekorRetry.OrderBy(kv => kv.Key, StringComparer.Ordinal))
|
|
{
|
|
builder.Append("attestor_rekor_retry_total{reason=\"");
|
|
builder.Append(EscapeLabelValue(key));
|
|
builder.Append("\"} ");
|
|
builder.Append(value.ToString(CultureInfo.InvariantCulture));
|
|
builder.Append('\n');
|
|
}
|
|
|
|
AppendHistogramOneLabel(
|
|
builder,
|
|
name: "rekor_inclusion_latency",
|
|
help: "Time to verify Rekor inclusion proof",
|
|
label: "success",
|
|
histograms: _rekorInclusionLatency);
|
|
|
|
return builder.ToString();
|
|
}
|
|
|
|
private static void AppendCounterHeader(StringBuilder builder, string name, string help)
|
|
{
|
|
builder.Append("# HELP ");
|
|
builder.Append(name);
|
|
builder.Append(' ');
|
|
builder.Append(help);
|
|
builder.Append('\n');
|
|
builder.Append("# TYPE ");
|
|
builder.Append(name);
|
|
builder.Append(" counter\n");
|
|
}
|
|
|
|
private static void AppendHistogramTwoLabels(
|
|
StringBuilder builder,
|
|
string name,
|
|
string help,
|
|
string labelA,
|
|
string labelB,
|
|
ConcurrentDictionary<TwoLabelKey, Histogram> histograms)
|
|
{
|
|
builder.Append("# HELP ");
|
|
builder.Append(name);
|
|
builder.Append(' ');
|
|
builder.Append(help);
|
|
builder.Append('\n');
|
|
builder.Append("# TYPE ");
|
|
builder.Append(name);
|
|
builder.Append(" histogram\n");
|
|
|
|
foreach (var grouping in histograms.OrderBy(kv => kv.Key.LabelA, StringComparer.Ordinal)
|
|
.ThenBy(kv => kv.Key.LabelB, StringComparer.Ordinal))
|
|
{
|
|
var labels = $"{labelA}=\"{EscapeLabelValue(grouping.Key.LabelA)}\",{labelB}=\"{EscapeLabelValue(grouping.Key.LabelB)}\"";
|
|
AppendHistogramSeries(builder, name, labels, grouping.Value.Snapshot());
|
|
}
|
|
}
|
|
|
|
private static void AppendHistogramOneLabel(
|
|
StringBuilder builder,
|
|
string name,
|
|
string help,
|
|
string label,
|
|
ConcurrentDictionary<string, Histogram> histograms)
|
|
{
|
|
builder.Append("# HELP ");
|
|
builder.Append(name);
|
|
builder.Append(' ');
|
|
builder.Append(help);
|
|
builder.Append('\n');
|
|
builder.Append("# TYPE ");
|
|
builder.Append(name);
|
|
builder.Append(" histogram\n");
|
|
|
|
foreach (var grouping in histograms.OrderBy(kv => kv.Key, StringComparer.Ordinal))
|
|
{
|
|
var labels = $"{label}=\"{EscapeLabelValue(grouping.Key)}\"";
|
|
AppendHistogramSeries(builder, name, labels, grouping.Value.Snapshot());
|
|
}
|
|
}
|
|
|
|
private static void AppendHistogramSeries(
|
|
StringBuilder builder,
|
|
string name,
|
|
string labels,
|
|
HistogramSnapshot snapshot)
|
|
{
|
|
long cumulative = 0;
|
|
|
|
for (var i = 0; i < snapshot.BucketUpperBounds.Length; i++)
|
|
{
|
|
cumulative += snapshot.BucketCounts[i];
|
|
builder.Append(name);
|
|
builder.Append("_bucket{");
|
|
builder.Append(labels);
|
|
builder.Append(",le=\"");
|
|
builder.Append(snapshot.BucketUpperBounds[i].ToString("G", CultureInfo.InvariantCulture));
|
|
builder.Append("\"} ");
|
|
builder.Append(cumulative.ToString(CultureInfo.InvariantCulture));
|
|
builder.Append('\n');
|
|
}
|
|
|
|
cumulative += snapshot.BucketCounts[^1];
|
|
builder.Append(name);
|
|
builder.Append("_bucket{");
|
|
builder.Append(labels);
|
|
builder.Append(",le=\"+Inf\"} ");
|
|
builder.Append(cumulative.ToString(CultureInfo.InvariantCulture));
|
|
builder.Append('\n');
|
|
|
|
builder.Append(name);
|
|
builder.Append("_sum{");
|
|
builder.Append(labels);
|
|
builder.Append("} ");
|
|
builder.Append(snapshot.SumSeconds.ToString("G", CultureInfo.InvariantCulture));
|
|
builder.Append('\n');
|
|
|
|
builder.Append(name);
|
|
builder.Append("_count{");
|
|
builder.Append(labels);
|
|
builder.Append("} ");
|
|
builder.Append(snapshot.Count.ToString(CultureInfo.InvariantCulture));
|
|
builder.Append('\n');
|
|
}
|
|
|
|
private static double ClampSeconds(double seconds)
|
|
=> double.IsNaN(seconds) || double.IsInfinity(seconds) || seconds < 0 ? 0 : seconds;
|
|
|
|
private static string NormalizeLabelValue(string? value, string fallback)
|
|
=> string.IsNullOrWhiteSpace(value) ? fallback : value.Trim();
|
|
|
|
private static string EscapeLabelValue(string value)
|
|
=> value.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal);
|
|
|
|
private sealed class Histogram
|
|
{
|
|
private readonly double[] _bucketUpperBounds;
|
|
private readonly long[] _bucketCounts;
|
|
private long _count;
|
|
private double _sumSeconds;
|
|
private readonly object _lock = new();
|
|
|
|
public Histogram(double[] bucketUpperBounds)
|
|
{
|
|
_bucketUpperBounds = bucketUpperBounds ?? throw new ArgumentNullException(nameof(bucketUpperBounds));
|
|
_bucketCounts = new long[_bucketUpperBounds.Length + 1];
|
|
}
|
|
|
|
public void Record(double seconds)
|
|
{
|
|
lock (_lock)
|
|
{
|
|
_count++;
|
|
_sumSeconds += seconds;
|
|
|
|
var bucketIndex = _bucketUpperBounds.Length;
|
|
for (var i = 0; i < _bucketUpperBounds.Length; i++)
|
|
{
|
|
if (seconds <= _bucketUpperBounds[i])
|
|
{
|
|
bucketIndex = i;
|
|
break;
|
|
}
|
|
}
|
|
|
|
_bucketCounts[bucketIndex]++;
|
|
}
|
|
}
|
|
|
|
public HistogramSnapshot Snapshot()
|
|
{
|
|
lock (_lock)
|
|
{
|
|
return new HistogramSnapshot(
|
|
(double[])_bucketUpperBounds.Clone(),
|
|
(long[])_bucketCounts.Clone(),
|
|
_count,
|
|
_sumSeconds);
|
|
}
|
|
}
|
|
}
|
|
|
|
private sealed record HistogramSnapshot(
|
|
double[] BucketUpperBounds,
|
|
long[] BucketCounts,
|
|
long Count,
|
|
double SumSeconds);
|
|
|
|
private sealed record ImportCounterKey(string TenantId, string Status);
|
|
|
|
private sealed record TwoLabelKey(string LabelA, string LabelB);
|
|
}
|
|
|