save progress
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,294 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
|
||||
namespace StellaOps.Scanner.WebService.Services;
|
||||
|
||||
internal sealed class OfflineKitMetricsStore
|
||||
{
|
||||
private static readonly double[] DefaultLatencyBucketsSeconds =
|
||||
{
|
||||
0.001,
|
||||
0.0025,
|
||||
0.005,
|
||||
0.01,
|
||||
0.025,
|
||||
0.05,
|
||||
0.1,
|
||||
0.25,
|
||||
0.5,
|
||||
1,
|
||||
2.5,
|
||||
5,
|
||||
10
|
||||
};
|
||||
|
||||
private readonly ConcurrentDictionary<ImportCounterKey, long> _imports = new();
|
||||
private readonly ConcurrentDictionary<TwoLabelKey, Histogram> _attestationVerifyLatency = new();
|
||||
private readonly ConcurrentDictionary<string, Histogram> _rekorInclusionLatency = new(StringComparer.Ordinal);
|
||||
private readonly ConcurrentDictionary<string, long> _rekorSuccess = new(StringComparer.Ordinal);
|
||||
private readonly ConcurrentDictionary<string, long> _rekorRetry = new(StringComparer.Ordinal);
|
||||
|
||||
public void RecordImport(string status, string tenantId)
|
||||
{
|
||||
status = NormalizeLabelValue(status, "unknown");
|
||||
tenantId = NormalizeLabelValue(tenantId, "unknown");
|
||||
_imports.AddOrUpdate(new ImportCounterKey(tenantId, status), 1, static (_, current) => current + 1);
|
||||
}
|
||||
|
||||
public void RecordAttestationVerifyLatency(string attestationType, double seconds, bool success)
|
||||
{
|
||||
attestationType = NormalizeLabelValue(attestationType, "unknown");
|
||||
seconds = ClampSeconds(seconds);
|
||||
var key = new TwoLabelKey(attestationType, success ? "true" : "false");
|
||||
var histogram = _attestationVerifyLatency.GetOrAdd(key, _ => new Histogram(DefaultLatencyBucketsSeconds));
|
||||
histogram.Record(seconds);
|
||||
}
|
||||
|
||||
public void RecordRekorSuccess(string mode)
|
||||
{
|
||||
mode = NormalizeLabelValue(mode, "unknown");
|
||||
_rekorSuccess.AddOrUpdate(mode, 1, static (_, current) => current + 1);
|
||||
}
|
||||
|
||||
public void RecordRekorRetry(string reason)
|
||||
{
|
||||
reason = NormalizeLabelValue(reason, "unknown");
|
||||
_rekorRetry.AddOrUpdate(reason, 1, static (_, current) => current + 1);
|
||||
}
|
||||
|
||||
public void RecordRekorInclusionLatency(double seconds, bool success)
|
||||
{
|
||||
seconds = ClampSeconds(seconds);
|
||||
var key = success ? "true" : "false";
|
||||
var histogram = _rekorInclusionLatency.GetOrAdd(key, _ => new Histogram(DefaultLatencyBucketsSeconds));
|
||||
histogram.Record(seconds);
|
||||
}
|
||||
|
||||
public string RenderPrometheus()
|
||||
{
|
||||
var builder = new StringBuilder(capacity: 4096);
|
||||
|
||||
AppendCounterHeader(builder, "offlinekit_import_total", "Total number of offline kit import attempts");
|
||||
foreach (var (key, value) in _imports.OrderBy(kv => kv.Key.TenantId, StringComparer.Ordinal)
|
||||
.ThenBy(kv => kv.Key.Status, StringComparer.Ordinal))
|
||||
{
|
||||
builder.Append("offlinekit_import_total{tenant_id=\"");
|
||||
builder.Append(EscapeLabelValue(key.TenantId));
|
||||
builder.Append("\",status=\"");
|
||||
builder.Append(EscapeLabelValue(key.Status));
|
||||
builder.Append("\"} ");
|
||||
builder.Append(value.ToString(CultureInfo.InvariantCulture));
|
||||
builder.Append('\n');
|
||||
}
|
||||
|
||||
AppendHistogramTwoLabels(
|
||||
builder,
|
||||
name: "offlinekit_attestation_verify_latency_seconds",
|
||||
help: "Time taken to verify attestations during import",
|
||||
labelA: "attestation_type",
|
||||
labelB: "success",
|
||||
histograms: _attestationVerifyLatency);
|
||||
|
||||
AppendCounterHeader(builder, "attestor_rekor_success_total", "Successful Rekor verification count");
|
||||
foreach (var (key, value) in _rekorSuccess.OrderBy(kv => kv.Key, StringComparer.Ordinal))
|
||||
{
|
||||
builder.Append("attestor_rekor_success_total{mode=\"");
|
||||
builder.Append(EscapeLabelValue(key));
|
||||
builder.Append("\"} ");
|
||||
builder.Append(value.ToString(CultureInfo.InvariantCulture));
|
||||
builder.Append('\n');
|
||||
}
|
||||
|
||||
AppendCounterHeader(builder, "attestor_rekor_retry_total", "Rekor verification retry count");
|
||||
foreach (var (key, value) in _rekorRetry.OrderBy(kv => kv.Key, StringComparer.Ordinal))
|
||||
{
|
||||
builder.Append("attestor_rekor_retry_total{reason=\"");
|
||||
builder.Append(EscapeLabelValue(key));
|
||||
builder.Append("\"} ");
|
||||
builder.Append(value.ToString(CultureInfo.InvariantCulture));
|
||||
builder.Append('\n');
|
||||
}
|
||||
|
||||
AppendHistogramOneLabel(
|
||||
builder,
|
||||
name: "rekor_inclusion_latency",
|
||||
help: "Time to verify Rekor inclusion proof",
|
||||
label: "success",
|
||||
histograms: _rekorInclusionLatency);
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
private static void AppendCounterHeader(StringBuilder builder, string name, string help)
|
||||
{
|
||||
builder.Append("# HELP ");
|
||||
builder.Append(name);
|
||||
builder.Append(' ');
|
||||
builder.Append(help);
|
||||
builder.Append('\n');
|
||||
builder.Append("# TYPE ");
|
||||
builder.Append(name);
|
||||
builder.Append(" counter\n");
|
||||
}
|
||||
|
||||
private static void AppendHistogramTwoLabels(
|
||||
StringBuilder builder,
|
||||
string name,
|
||||
string help,
|
||||
string labelA,
|
||||
string labelB,
|
||||
ConcurrentDictionary<TwoLabelKey, Histogram> histograms)
|
||||
{
|
||||
builder.Append("# HELP ");
|
||||
builder.Append(name);
|
||||
builder.Append(' ');
|
||||
builder.Append(help);
|
||||
builder.Append('\n');
|
||||
builder.Append("# TYPE ");
|
||||
builder.Append(name);
|
||||
builder.Append(" histogram\n");
|
||||
|
||||
foreach (var grouping in histograms.OrderBy(kv => kv.Key.LabelA, StringComparer.Ordinal)
|
||||
.ThenBy(kv => kv.Key.LabelB, StringComparer.Ordinal))
|
||||
{
|
||||
var labels = $"{labelA}=\"{EscapeLabelValue(grouping.Key.LabelA)}\",{labelB}=\"{EscapeLabelValue(grouping.Key.LabelB)}\"";
|
||||
AppendHistogramSeries(builder, name, labels, grouping.Value.Snapshot());
|
||||
}
|
||||
}
|
||||
|
||||
private static void AppendHistogramOneLabel(
|
||||
StringBuilder builder,
|
||||
string name,
|
||||
string help,
|
||||
string label,
|
||||
ConcurrentDictionary<string, Histogram> histograms)
|
||||
{
|
||||
builder.Append("# HELP ");
|
||||
builder.Append(name);
|
||||
builder.Append(' ');
|
||||
builder.Append(help);
|
||||
builder.Append('\n');
|
||||
builder.Append("# TYPE ");
|
||||
builder.Append(name);
|
||||
builder.Append(" histogram\n");
|
||||
|
||||
foreach (var grouping in histograms.OrderBy(kv => kv.Key, StringComparer.Ordinal))
|
||||
{
|
||||
var labels = $"{label}=\"{EscapeLabelValue(grouping.Key)}\"";
|
||||
AppendHistogramSeries(builder, name, labels, grouping.Value.Snapshot());
|
||||
}
|
||||
}
|
||||
|
||||
private static void AppendHistogramSeries(
|
||||
StringBuilder builder,
|
||||
string name,
|
||||
string labels,
|
||||
HistogramSnapshot snapshot)
|
||||
{
|
||||
long cumulative = 0;
|
||||
|
||||
for (var i = 0; i < snapshot.BucketUpperBounds.Length; i++)
|
||||
{
|
||||
cumulative += snapshot.BucketCounts[i];
|
||||
builder.Append(name);
|
||||
builder.Append("_bucket{");
|
||||
builder.Append(labels);
|
||||
builder.Append(",le=\"");
|
||||
builder.Append(snapshot.BucketUpperBounds[i].ToString("G", CultureInfo.InvariantCulture));
|
||||
builder.Append("\"} ");
|
||||
builder.Append(cumulative.ToString(CultureInfo.InvariantCulture));
|
||||
builder.Append('\n');
|
||||
}
|
||||
|
||||
cumulative += snapshot.BucketCounts[^1];
|
||||
builder.Append(name);
|
||||
builder.Append("_bucket{");
|
||||
builder.Append(labels);
|
||||
builder.Append(",le=\"+Inf\"} ");
|
||||
builder.Append(cumulative.ToString(CultureInfo.InvariantCulture));
|
||||
builder.Append('\n');
|
||||
|
||||
builder.Append(name);
|
||||
builder.Append("_sum{");
|
||||
builder.Append(labels);
|
||||
builder.Append("} ");
|
||||
builder.Append(snapshot.SumSeconds.ToString("G", CultureInfo.InvariantCulture));
|
||||
builder.Append('\n');
|
||||
|
||||
builder.Append(name);
|
||||
builder.Append("_count{");
|
||||
builder.Append(labels);
|
||||
builder.Append("} ");
|
||||
builder.Append(snapshot.Count.ToString(CultureInfo.InvariantCulture));
|
||||
builder.Append('\n');
|
||||
}
|
||||
|
||||
private static double ClampSeconds(double seconds)
|
||||
=> double.IsNaN(seconds) || double.IsInfinity(seconds) || seconds < 0 ? 0 : seconds;
|
||||
|
||||
private static string NormalizeLabelValue(string? value, string fallback)
|
||||
=> string.IsNullOrWhiteSpace(value) ? fallback : value.Trim();
|
||||
|
||||
private static string EscapeLabelValue(string value)
|
||||
=> value.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal);
|
||||
|
||||
private sealed class Histogram
|
||||
{
|
||||
private readonly double[] _bucketUpperBounds;
|
||||
private readonly long[] _bucketCounts;
|
||||
private long _count;
|
||||
private double _sumSeconds;
|
||||
private readonly object _lock = new();
|
||||
|
||||
public Histogram(double[] bucketUpperBounds)
|
||||
{
|
||||
_bucketUpperBounds = bucketUpperBounds ?? throw new ArgumentNullException(nameof(bucketUpperBounds));
|
||||
_bucketCounts = new long[_bucketUpperBounds.Length + 1];
|
||||
}
|
||||
|
||||
public void Record(double seconds)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
_count++;
|
||||
_sumSeconds += seconds;
|
||||
|
||||
var bucketIndex = _bucketUpperBounds.Length;
|
||||
for (var i = 0; i < _bucketUpperBounds.Length; i++)
|
||||
{
|
||||
if (seconds <= _bucketUpperBounds[i])
|
||||
{
|
||||
bucketIndex = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
_bucketCounts[bucketIndex]++;
|
||||
}
|
||||
}
|
||||
|
||||
public HistogramSnapshot Snapshot()
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
return new HistogramSnapshot(
|
||||
(double[])_bucketUpperBounds.Clone(),
|
||||
(long[])_bucketCounts.Clone(),
|
||||
_count,
|
||||
_sumSeconds);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record HistogramSnapshot(
|
||||
double[] BucketUpperBounds,
|
||||
long[] BucketCounts,
|
||||
long Count,
|
||||
double SumSeconds);
|
||||
|
||||
private sealed record ImportCounterKey(string TenantId, string Status);
|
||||
|
||||
private sealed record TwoLabelKey(string LabelA, string LabelB);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user