feat: Add VEX compact fixture and implement offline verifier for Findings Ledger exports
- Introduced a new VEX compact fixture for testing purposes. - Implemented `verify_export.py` script to validate Findings Ledger exports, ensuring deterministic ordering and applying redaction manifests. - Added a lightweight stub `HarnessRunner` for unit tests to validate ledger hashing expectations. - Documented tasks related to the Mirror Creator. - Created models for entropy signals and implemented the `EntropyPenaltyCalculator` to compute penalties based on scanner outputs. - Developed unit tests for `EntropyPenaltyCalculator` to ensure correct penalty calculations and handling of edge cases. - Added tests for symbol ID normalization in the reachability scanner. - Enhanced console status service with comprehensive unit tests for connection handling and error recovery. - Included Cosign tool version 2.6.0 with checksums for various platforms.
This commit is contained in:
StellaOps Bot
@@ -1,5 +1,7 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics.Metrics;
|
||||
using System.Reflection;
|
||||
|
||||
namespace StellaOps.Findings.Ledger.Observability;
|
||||
|
||||
@@ -22,6 +24,14 @@ internal static class LedgerMetrics
|
||||
"ledger_events_total",
|
||||
description: "Number of ledger events appended.");
|
||||
|
||||
private static readonly Counter<long> BackpressureApplied = Meter.CreateCounter<long>(
|
||||
"ledger_backpressure_applied_total",
|
||||
description: "Times ingest backpressure thresholds were exceeded.");
|
||||
|
||||
private static readonly Counter<long> QuotaRejections = Meter.CreateCounter<long>(
|
||||
"ledger_quota_rejections_total",
|
||||
description: "Requests rejected due to configured quotas.");
|
||||
|
||||
private static readonly Histogram<double> ProjectionApplySeconds = Meter.CreateHistogram<double>(
|
||||
"ledger_projection_apply_seconds",
|
||||
unit: "s",
|
||||
@@ -45,21 +55,38 @@ internal static class LedgerMetrics
|
||||
"ledger_merkle_anchor_failures_total",
|
||||
description: "Count of Merkle anchor failures by reason.");
|
||||
|
||||
private static readonly Counter<long> AttachmentsEncryptionFailures = Meter.CreateCounter<long>(
|
||||
"ledger_attachments_encryption_failures_total",
|
||||
description: "Count of attachment encryption/signing/upload failures.");
|
||||
|
||||
private static readonly ObservableGauge<double> ProjectionLagGauge =
|
||||
Meter.CreateObservableGauge("ledger_projection_lag_seconds", ObserveProjectionLag, unit: "s",
|
||||
description: "Lag between ledger recorded_at and projection application time.");
|
||||
|
||||
private static readonly ObservableGauge<long> IngestBacklogGauge =
|
||||
Meter.CreateObservableGauge("ledger_ingest_backlog_events", ObserveBacklog,
|
||||
description: "Number of events buffered for ingestion/anchoring.");
|
||||
description: "Number of events buffered for ingestion/anchoring per tenant.");
|
||||
|
||||
private static readonly ObservableGauge<long> QuotaRemainingGauge =
|
||||
Meter.CreateObservableGauge("ledger_quota_remaining", ObserveQuotaRemaining,
|
||||
description: "Remaining ingest backlog capacity before backpressure applies.");
|
||||
|
||||
private static readonly ObservableGauge<long> DbConnectionsGauge =
|
||||
Meter.CreateObservableGauge("ledger_db_connections_active", ObserveDbConnections,
|
||||
description: "Active PostgreSQL connections by role.");
|
||||
|
||||
private static readonly ObservableGauge<long> AppVersionGauge =
|
||||
Meter.CreateObservableGauge("ledger_app_version_info", ObserveAppVersion,
|
||||
description: "Static gauge exposing build version and git sha.");
|
||||
|
||||
private static readonly ConcurrentDictionary<string, double> ProjectionLagByTenant = new(StringComparer.Ordinal);
|
||||
private static readonly ConcurrentDictionary<string, long> DbConnectionsByRole = new(StringComparer.OrdinalIgnoreCase);
|
||||
private static long _ingestBacklog;
|
||||
private static readonly ConcurrentDictionary<string, long> BacklogByTenant = new(StringComparer.Ordinal);
|
||||
|
||||
private static long _ingestBacklogLimit = 5000;
|
||||
|
||||
private static readonly string AppVersion = Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "0.0.0";
|
||||
private static readonly string GitSha = Environment.GetEnvironmentVariable("GIT_SHA") ?? "unknown";
|
||||
|
||||
public static void RecordWriteSuccess(TimeSpan duration, string? tenantId, string? eventType, string? source)
|
||||
{
|
||||
@@ -127,17 +154,55 @@ internal static class LedgerMetrics
|
||||
MerkleAnchorFailures.Add(1, tags);
|
||||
}
|
||||
|
||||
public static void IncrementBacklog() => Interlocked.Increment(ref _ingestBacklog);
|
||||
|
||||
public static void DecrementBacklog()
|
||||
public static void RecordAttachmentFailure(string tenantId, string stage)
|
||||
{
|
||||
var value = Interlocked.Decrement(ref _ingestBacklog);
|
||||
if (value < 0)
|
||||
var tags = new KeyValuePair<string, object?>[]
|
||||
{
|
||||
Interlocked.Exchange(ref _ingestBacklog, 0);
|
||||
new("tenant", tenantId),
|
||||
new("stage", stage)
|
||||
};
|
||||
AttachmentsEncryptionFailures.Add(1, tags);
|
||||
}
|
||||
|
||||
public static void ConfigureQuotas(long ingestBacklogLimit)
|
||||
{
|
||||
if (ingestBacklogLimit > 0)
|
||||
{
|
||||
Interlocked.Exchange(ref _ingestBacklogLimit, ingestBacklogLimit);
|
||||
}
|
||||
}
|
||||
|
||||
public static long IncrementBacklog(string? tenantId = null)
|
||||
{
|
||||
var key = NormalizeTenant(tenantId);
|
||||
var backlog = BacklogByTenant.AddOrUpdate(key, _ => 1, (_, current) => current + 1);
|
||||
if (backlog > _ingestBacklogLimit)
|
||||
{
|
||||
BackpressureApplied.Add(1, new KeyValuePair<string, object?>[]
|
||||
{
|
||||
new("tenant", key),
|
||||
new("reason", "ingest_backlog"),
|
||||
new("limit", _ingestBacklogLimit)
|
||||
});
|
||||
}
|
||||
return backlog;
|
||||
}
|
||||
|
||||
public static void RecordQuotaRejection(string tenantId, string reason)
|
||||
{
|
||||
QuotaRejections.Add(1, new KeyValuePair<string, object?>[]
|
||||
{
|
||||
new("tenant", NormalizeTenant(tenantId)),
|
||||
new("reason", reason)
|
||||
});
|
||||
}
|
||||
|
||||
public static void DecrementBacklog(string? tenantId = null)
|
||||
{
|
||||
var key = NormalizeTenant(tenantId);
|
||||
BacklogByTenant.AddOrUpdate(key, _ => 0, (_, current) => Math.Max(0, current - 1));
|
||||
}
|
||||
|
||||
public static void ConnectionOpened(string role)
|
||||
{
|
||||
var normalized = NormalizeRole(role);
|
||||
@@ -150,12 +215,19 @@ internal static class LedgerMetrics
|
||||
DbConnectionsByRole.AddOrUpdate(normalized, _ => 0, (_, current) => Math.Max(0, current - 1));
|
||||
}
|
||||
|
||||
public static void IncrementDbConnection(string role) => ConnectionOpened(role);
|
||||
|
||||
public static void DecrementDbConnection(string role) => ConnectionClosed(role);
|
||||
|
||||
public static void UpdateProjectionLag(string? tenantId, double lagSeconds)
|
||||
{
|
||||
var key = string.IsNullOrWhiteSpace(tenantId) ? string.Empty : tenantId;
|
||||
ProjectionLagByTenant[key] = lagSeconds < 0 ? 0 : lagSeconds;
|
||||
}
|
||||
|
||||
public static void RecordProjectionLag(TimeSpan lag, string? tenantId) =>
|
||||
UpdateProjectionLag(tenantId, lag.TotalSeconds);
|
||||
|
||||
private static IEnumerable<Measurement<double>> ObserveProjectionLag()
|
||||
{
|
||||
foreach (var kvp in ProjectionLagByTenant)
|
||||
@@ -166,7 +238,19 @@ internal static class LedgerMetrics
|
||||
|
||||
private static IEnumerable<Measurement<long>> ObserveBacklog()
|
||||
{
|
||||
yield return new Measurement<long>(Interlocked.Read(ref _ingestBacklog));
|
||||
foreach (var kvp in BacklogByTenant)
|
||||
{
|
||||
yield return new Measurement<long>(kvp.Value, new KeyValuePair<string, object?>("tenant", kvp.Key));
|
||||
}
|
||||
}
|
||||
|
||||
private static IEnumerable<Measurement<long>> ObserveQuotaRemaining()
|
||||
{
|
||||
foreach (var kvp in BacklogByTenant)
|
||||
{
|
||||
var remaining = Math.Max(0, _ingestBacklogLimit - kvp.Value);
|
||||
yield return new Measurement<long>(remaining, new KeyValuePair<string, object?>("tenant", kvp.Key));
|
||||
}
|
||||
}
|
||||
|
||||
private static IEnumerable<Measurement<long>> ObserveDbConnections()
|
||||
@@ -177,5 +261,13 @@ internal static class LedgerMetrics
|
||||
}
|
||||
}
|
||||
|
||||
private static IEnumerable<Measurement<long>> ObserveAppVersion()
|
||||
{
|
||||
yield return new Measurement<long>(1, new KeyValuePair<string, object?>("version", AppVersion),
|
||||
new KeyValuePair<string, object?>("git_sha", GitSha));
|
||||
}
|
||||
|
||||
private static string NormalizeRole(string role) => string.IsNullOrWhiteSpace(role) ? "unspecified" : role.ToLowerInvariant();
|
||||
|
||||
private static string NormalizeTenant(string? tenantId) => string.IsNullOrWhiteSpace(tenantId) ? string.Empty : tenantId;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user