stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

up

Browse Source
This commit is contained in:
StellaOps Bot
2025-12-09 00:20:52 +02:00
parent 3d01bf9edc
commit bc0762e97d
261 changed files with 14033 additions and 4427 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/Concelier/Directory.Build.props
View File

@@ -9,6 +9,7 @@
<!-- Concelier is migrating off MongoDB; strip implicit Mongo2Go/Mongo driver packages inherited from the repo root. -->
<PackageReference Remove="Mongo2Go" />
<PackageReference Remove="MongoDB.Driver" />
<PackageReference Remove="MongoDB.Bson" />
</ItemGroup>
<ItemGroup Condition="$([System.String]::Copy('$(MSBuildProjectName)').EndsWith('.Tests')) and '$(UseConcelierTestInfra)'=='true'">
<PackageReference Include="coverlet.collector" Version="6.0.4" />

14
src/Concelier/StellaOps.Concelier.WebService/AGENTS.md
View File

@@ -2,13 +2,13 @@
## Role
Minimal API host wiring configuration, storage, plugin routines, and job endpoints. Operational surface for health, readiness, and job control.
## Scope
- Configuration: appsettings.json + etc/concelier.yaml (yaml path = ../etc/concelier.yaml); bind into ConcelierOptions with validation (Only Mongo supported).
- Mongo: MongoUrl from options.Storage.Dsn; IMongoClient/IMongoDatabase singletons; default database name fallback (options -> URL -> "concelier").
- Services: AddMongoStorage(); AddSourceHttpClients(); RegisterPluginRoutines(configuration, PluginHostOptions).
- Bootstrap: MongoBootstrapper.InitializeAsync on startup.
- Configuration: appsettings.json + etc/concelier.yaml (yaml path = ../etc/concelier.yaml); bind into ConcelierOptions with PostgreSQL storage enabled by default.
- Storage: PostgreSQL only (`Concelier:PostgresStorage:*`). No MongoDB/Mongo2Go; readiness probes issue `SELECT 1` against ConcelierDataSource.
- Services: AddConcelierPostgresStorage(); AddSourceHttpClients(); RegisterPluginRoutines(configuration, PluginHostOptions).
- Bootstrap: PostgreSQL connectivity verified on startup.
- Endpoints (configuration & job control only; root path intentionally unbound):
- GET /health -> {status:"healthy"} after options validation binds.
- GET /ready -> MongoDB ping; 503 on MongoException/Timeout.
- GET /ready -> PostgreSQL connectivity check; degraded if connection fails.
- GET /jobs?kind=&limit= -> recent runs.
- GET /jobs/{id} -> run detail.
- GET /jobs/definitions -> definitions with lastRun.
@@ -18,7 +18,7 @@ Minimal API host wiring configuration, storage, plugin routines, and job endpoin
- POST /jobs/{*jobKind} with {trigger?,parameters?} -> 202 Accepted (Location:/jobs/{runId}) | 404 | 409 | 423.
- PluginHost defaults: BaseDirectory = solution root; PluginsDirectory = "StellaOps.Concelier.PluginBinaries"; SearchPatterns += "StellaOps.Concelier.Plugin.*.dll"; EnsureDirectoryExists = true.
## Participants
- Core job system; Storage.Mongo; Source.Common HTTP clients; Exporter and Connector plugin routines discover/register jobs.
- Core job system; Storage.Postgres; Source.Common HTTP clients; Exporter and Connector plugin routines discover/register jobs.
## Interfaces & contracts
- Dependency injection boundary for all connectors/exporters; IOptions<ConcelierOptions> validated on start.
- Cancellation: pass app.Lifetime.ApplicationStopping to bootstrapper.
@@ -30,7 +30,7 @@ Out: business logic of jobs, HTML UI, authn/z (future).
- Structured responses with status codes; no stack traces in HTTP bodies; errors mapped cleanly.
## Tests
- Author and review coverage in `../StellaOps.Concelier.WebService.Tests`.
- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`.
- Shared fixtures (PostgreSQL-backed harnesses) live in `../StellaOps.Concelier.Testing`.
- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios.
## Required Reading

21
src/Concelier/StellaOps.Concelier.WebService/Diagnostics/HealthContracts.cs
View File

@@ -1,10 +1,11 @@
namespace StellaOps.Concelier.WebService.Diagnostics;
internal sealed record StorageBootstrapHealth(
string Driver,
bool Completed,
DateTimeOffset? CompletedAt,
double? DurationMs);
internal sealed record StorageHealth(
string Backend,
bool Ready,
DateTimeOffset? CheckedAt,
double? LatencyMs,
string? Error);
internal sealed record TelemetryHealth(
bool Enabled,
@@ -16,17 +17,11 @@ internal sealed record HealthDocument(
string Status,
DateTimeOffset StartedAt,
double UptimeSeconds,
StorageBootstrapHealth Storage,
StorageHealth Storage,
TelemetryHealth Telemetry);
internal sealed record MongoReadyHealth(
string Status,
double? LatencyMs,
DateTimeOffset? CheckedAt,
string? Error);
internal sealed record ReadyDocument(
string Status,
DateTimeOffset StartedAt,
double UptimeSeconds,
MongoReadyHealth Mongo);
StorageHealth Storage);

22
src/Concelier/StellaOps.Concelier.WebService/Diagnostics/ServiceStatus.cs
View File

@@ -11,8 +11,8 @@ internal sealed class ServiceStatus
private DateTimeOffset? _bootstrapCompletedAt;
private TimeSpan? _bootstrapDuration;
private DateTimeOffset? _lastReadyCheckAt;
private TimeSpan? _lastMongoLatency;
private string? _lastMongoError;
private TimeSpan? _lastStorageLatency;
private string? _lastStorageError;
private bool _lastReadySucceeded;
public ServiceStatus(TimeProvider timeProvider)
@@ -31,8 +31,8 @@ internal sealed class ServiceStatus
BootstrapCompletedAt: _bootstrapCompletedAt,
BootstrapDuration: _bootstrapDuration,
LastReadyCheckAt: _lastReadyCheckAt,
LastMongoLatency: _lastMongoLatency,
LastMongoError: _lastMongoError,
LastStorageLatency: _lastStorageLatency,
LastStorageError: _lastStorageError,
LastReadySucceeded: _lastReadySucceeded);
}
}
@@ -45,19 +45,19 @@ internal sealed class ServiceStatus
_bootstrapCompletedAt = completedAt;
_bootstrapDuration = duration;
_lastReadySucceeded = true;
_lastMongoLatency = duration;
_lastMongoError = null;
_lastStorageLatency = duration;
_lastStorageError = null;
_lastReadyCheckAt = completedAt;
}
}
public void RecordMongoCheck(bool success, TimeSpan latency, string? error)
public void RecordStorageCheck(bool success, TimeSpan latency, string? error)
{
lock (_sync)
{
_lastReadySucceeded = success;
_lastMongoLatency = latency;
_lastMongoError = success ? null : error;
_lastStorageLatency = latency;
_lastStorageError = success ? null : error;
_lastReadyCheckAt = _timeProvider.GetUtcNow();
}
}
@@ -69,6 +69,6 @@ internal sealed record ServiceHealthSnapshot(
DateTimeOffset? BootstrapCompletedAt,
TimeSpan? BootstrapDuration,
DateTimeOffset? LastReadyCheckAt,
TimeSpan? LastMongoLatency,
string? LastMongoError,
TimeSpan? LastStorageLatency,
string? LastStorageError,
bool LastReadySucceeded);

393
src/Concelier/StellaOps.Concelier.WebService/Extensions/TelemetryExtensions.cs
View File

@@ -1,71 +1,71 @@
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Reflection;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.DependencyInjection;
using OpenTelemetry.Metrics;
using OpenTelemetry.Resources;
using OpenTelemetry.Trace;
using Serilog;
using Serilog.Core;
using Serilog.Events;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Reflection;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.DependencyInjection;
using OpenTelemetry.Metrics;
using OpenTelemetry.Resources;
using OpenTelemetry.Trace;
using Serilog;
using Serilog.Core;
using Serilog.Events;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.Common.Telemetry;
using StellaOps.Concelier.WebService.Diagnostics;
using StellaOps.Concelier.WebService.Options;
using StellaOps.Ingestion.Telemetry;
namespace StellaOps.Concelier.WebService.Extensions;
public static class TelemetryExtensions
{
public static void ConfigureConcelierTelemetry(this WebApplicationBuilder builder, ConcelierOptions options)
{
ArgumentNullException.ThrowIfNull(builder);
ArgumentNullException.ThrowIfNull(options);
var telemetry = options.Telemetry ?? new ConcelierOptions.TelemetryOptions();
if (telemetry.EnableLogging)
{
builder.Host.UseSerilog((context, services, configuration) =>
{
ConfigureSerilog(configuration, telemetry, builder.Environment.EnvironmentName, builder.Environment.ApplicationName);
});
}
if (!telemetry.Enabled || (!telemetry.EnableTracing && !telemetry.EnableMetrics))
{
return;
}
var openTelemetry = builder.Services.AddOpenTelemetry();
openTelemetry.ConfigureResource(resource =>
{
var serviceName = telemetry.ServiceName ?? builder.Environment.ApplicationName;
var version = Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown";
resource.AddService(serviceName, serviceVersion: version, serviceInstanceId: Environment.MachineName);
resource.AddAttributes(new[]
{
new KeyValuePair<string, object>("deployment.environment", builder.Environment.EnvironmentName),
});
foreach (var attribute in telemetry.ResourceAttributes)
{
if (string.IsNullOrWhiteSpace(attribute.Key) || attribute.Value is null)
{
continue;
}
resource.AddAttributes(new[] { new KeyValuePair<string, object>(attribute.Key, attribute.Value) });
}
});
if (telemetry.EnableTracing)
{
namespace StellaOps.Concelier.WebService.Extensions;
public static class TelemetryExtensions
{
public static void ConfigureConcelierTelemetry(this WebApplicationBuilder builder, ConcelierOptions options)
{
ArgumentNullException.ThrowIfNull(builder);
ArgumentNullException.ThrowIfNull(options);
var telemetry = options.Telemetry ?? new ConcelierOptions.TelemetryOptions();
if (telemetry.EnableLogging)
{
builder.Host.UseSerilog((context, services, configuration) =>
{
ConfigureSerilog(configuration, telemetry, builder.Environment.EnvironmentName, builder.Environment.ApplicationName);
});
}
if (!telemetry.Enabled || (!telemetry.EnableTracing && !telemetry.EnableMetrics))
{
return;
}
var openTelemetry = builder.Services.AddOpenTelemetry();
openTelemetry.ConfigureResource(resource =>
{
var serviceName = telemetry.ServiceName ?? builder.Environment.ApplicationName;
var version = Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown";
resource.AddService(serviceName, serviceVersion: version, serviceInstanceId: Environment.MachineName);
resource.AddAttributes(new[]
{
new KeyValuePair<string, object>("deployment.environment", builder.Environment.EnvironmentName),
});
foreach (var attribute in telemetry.ResourceAttributes)
{
if (string.IsNullOrWhiteSpace(attribute.Key) || attribute.Value is null)
{
continue;
}
resource.AddAttributes(new[] { new KeyValuePair<string, object>(attribute.Key, attribute.Value) });
}
});
if (telemetry.EnableTracing)
{
openTelemetry.WithTracing(tracing =>
{
tracing
@@ -74,15 +74,15 @@ public static class TelemetryExtensions
.AddSource(IngestionTelemetry.ActivitySourceName)
.AddAspNetCoreInstrumentation()
.AddHttpClientInstrumentation();
ConfigureExporters(telemetry, tracing);
});
}
if (telemetry.EnableMetrics)
{
openTelemetry.WithMetrics(metrics =>
{
ConfigureExporters(telemetry, tracing);
});
}
if (telemetry.EnableMetrics)
{
openTelemetry.WithMetrics(metrics =>
{
metrics
.AddMeter(JobDiagnostics.MeterName)
.AddMeter(SourceDiagnostics.MeterName)
@@ -92,131 +92,132 @@ public static class TelemetryExtensions
.AddMeter("StellaOps.Concelier.Connector.Vndr.Chromium")
.AddMeter("StellaOps.Concelier.Connector.Vndr.Apple")
.AddMeter("StellaOps.Concelier.Connector.Vndr.Adobe")
.AddMeter("StellaOps.Concelier.VulnExplorer")
.AddMeter(JobMetrics.MeterName)
.AddAspNetCoreInstrumentation()
.AddHttpClientInstrumentation()
.AddRuntimeInstrumentation();
ConfigureExporters(telemetry, metrics);
});
}
}
private static void ConfigureSerilog(LoggerConfiguration configuration, ConcelierOptions.TelemetryOptions telemetry, string environmentName, string applicationName)
{
if (!Enum.TryParse(telemetry.MinimumLogLevel, ignoreCase: true, out LogEventLevel level))
{
level = LogEventLevel.Information;
}
configuration
.MinimumLevel.Is(level)
.MinimumLevel.Override("Microsoft", LogEventLevel.Warning)
.MinimumLevel.Override("Microsoft.Hosting.Lifetime", LogEventLevel.Information)
.Enrich.FromLogContext()
.Enrich.With<ActivityEnricher>()
.Enrich.WithProperty("service.name", telemetry.ServiceName ?? applicationName)
.Enrich.WithProperty("deployment.environment", environmentName)
.WriteTo.Console(outputTemplate: "[{Timestamp:O}] [{Level:u3}] {Message:lj} {Properties}{NewLine}{Exception}");
}
private static void ConfigureExporters(ConcelierOptions.TelemetryOptions telemetry, TracerProviderBuilder tracing)
{
if (string.IsNullOrWhiteSpace(telemetry.OtlpEndpoint))
{
if (telemetry.ExportConsole)
{
tracing.AddConsoleExporter();
}
return;
}
tracing.AddOtlpExporter(options =>
{
options.Endpoint = new Uri(telemetry.OtlpEndpoint);
var headers = BuildHeaders(telemetry);
if (!string.IsNullOrEmpty(headers))
{
options.Headers = headers;
}
});
if (telemetry.ExportConsole)
{
tracing.AddConsoleExporter();
}
}
private static void ConfigureExporters(ConcelierOptions.TelemetryOptions telemetry, MeterProviderBuilder metrics)
{
if (string.IsNullOrWhiteSpace(telemetry.OtlpEndpoint))
{
if (telemetry.ExportConsole)
{
metrics.AddConsoleExporter();
}
return;
}
metrics.AddOtlpExporter(options =>
{
options.Endpoint = new Uri(telemetry.OtlpEndpoint);
var headers = BuildHeaders(telemetry);
if (!string.IsNullOrEmpty(headers))
{
options.Headers = headers;
}
});
if (telemetry.ExportConsole)
{
metrics.AddConsoleExporter();
}
}
private static string? BuildHeaders(ConcelierOptions.TelemetryOptions telemetry)
{
if (telemetry.OtlpHeaders.Count == 0)
{
return null;
}
return string.Join(",", telemetry.OtlpHeaders
.Where(static kvp => !string.IsNullOrWhiteSpace(kvp.Key) && !string.IsNullOrWhiteSpace(kvp.Value))
.Select(static kvp => $"{kvp.Key}={kvp.Value}"));
}
}
internal sealed class ActivityEnricher : ILogEventEnricher
{
public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory)
{
var activity = Activity.Current;
if (activity is null)
{
return;
}
if (activity.TraceId != default)
{
logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty("trace_id", activity.TraceId.ToString()));
}
if (activity.SpanId != default)
{
logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty("span_id", activity.SpanId.ToString()));
}
if (activity.ParentSpanId != default)
{
logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty("parent_span_id", activity.ParentSpanId.ToString()));
}
if (!string.IsNullOrEmpty(activity.TraceStateString))
{
logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty("trace_state", activity.TraceStateString));
}
}
}
ConfigureExporters(telemetry, metrics);
});
}
}
private static void ConfigureSerilog(LoggerConfiguration configuration, ConcelierOptions.TelemetryOptions telemetry, string environmentName, string applicationName)
{
if (!Enum.TryParse(telemetry.MinimumLogLevel, ignoreCase: true, out LogEventLevel level))
{
level = LogEventLevel.Information;
}
configuration
.MinimumLevel.Is(level)
.MinimumLevel.Override("Microsoft", LogEventLevel.Warning)
.MinimumLevel.Override("Microsoft.Hosting.Lifetime", LogEventLevel.Information)
.Enrich.FromLogContext()
.Enrich.With<ActivityEnricher>()
.Enrich.WithProperty("service.name", telemetry.ServiceName ?? applicationName)
.Enrich.WithProperty("deployment.environment", environmentName)
.WriteTo.Console(outputTemplate: "[{Timestamp:O}] [{Level:u3}] {Message:lj} {Properties}{NewLine}{Exception}");
}
private static void ConfigureExporters(ConcelierOptions.TelemetryOptions telemetry, TracerProviderBuilder tracing)
{
if (string.IsNullOrWhiteSpace(telemetry.OtlpEndpoint))
{
if (telemetry.ExportConsole)
{
tracing.AddConsoleExporter();
}
return;
}
tracing.AddOtlpExporter(options =>
{
options.Endpoint = new Uri(telemetry.OtlpEndpoint);
var headers = BuildHeaders(telemetry);
if (!string.IsNullOrEmpty(headers))
{
options.Headers = headers;
}
});
if (telemetry.ExportConsole)
{
tracing.AddConsoleExporter();
}
}
private static void ConfigureExporters(ConcelierOptions.TelemetryOptions telemetry, MeterProviderBuilder metrics)
{
if (string.IsNullOrWhiteSpace(telemetry.OtlpEndpoint))
{
if (telemetry.ExportConsole)
{
metrics.AddConsoleExporter();
}
return;
}
metrics.AddOtlpExporter(options =>
{
options.Endpoint = new Uri(telemetry.OtlpEndpoint);
var headers = BuildHeaders(telemetry);
if (!string.IsNullOrEmpty(headers))
{
options.Headers = headers;
}
});
if (telemetry.ExportConsole)
{
metrics.AddConsoleExporter();
}
}
private static string? BuildHeaders(ConcelierOptions.TelemetryOptions telemetry)
{
if (telemetry.OtlpHeaders.Count == 0)
{
return null;
}
return string.Join(",", telemetry.OtlpHeaders
.Where(static kvp => !string.IsNullOrWhiteSpace(kvp.Key) && !string.IsNullOrWhiteSpace(kvp.Value))
.Select(static kvp => $"{kvp.Key}={kvp.Value}"));
}
}
internal sealed class ActivityEnricher : ILogEventEnricher
{
public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory)
{
var activity = Activity.Current;
if (activity is null)
{
return;
}
if (activity.TraceId != default)
{
logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty("trace_id", activity.TraceId.ToString()));
}
if (activity.SpanId != default)
{
logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty("span_id", activity.SpanId.ToString()));
}
if (activity.ParentSpanId != default)
{
logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty("parent_span_id", activity.ParentSpanId.ToString()));
}
if (!string.IsNullOrEmpty(activity.TraceStateString))
{
logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty("trace_state", activity.TraceStateString));
}
}
}

7
src/Concelier/StellaOps.Concelier.WebService/Options/ConcelierOptions.cs
View File

@@ -7,9 +7,13 @@ namespace StellaOps.Concelier.WebService.Options;
public sealed class ConcelierOptions
{
[Obsolete("Mongo storage has been removed; use PostgresStorage.")]
public StorageOptions Storage { get; set; } = new();
public PostgresStorageOptions? PostgresStorage { get; set; }
public PostgresStorageOptions? PostgresStorage { get; set; } = new PostgresStorageOptions
{
Enabled = true
};
public PluginOptions Plugins { get; set; } = new();
@@ -33,6 +37,7 @@ public sealed class ConcelierOptions
/// </summary>
public AirGapOptions AirGap { get; set; } = new();
[Obsolete("Mongo storage has been removed; use PostgresStorage.")]
public sealed class StorageOptions
{
public string Driver { get; set; } = "mongo";

119
src/Concelier/StellaOps.Concelier.WebService/Options/ConcelierOptionsValidator.cs
View File

@@ -2,30 +2,17 @@ using System;
using System.Collections.Generic;
using Microsoft.Extensions.Logging;
using StellaOps.Auth.Abstractions;
namespace StellaOps.Concelier.WebService.Options;
public static class ConcelierOptionsValidator
{
public static void Validate(ConcelierOptions options)
{
ArgumentNullException.ThrowIfNull(options);
if (!string.Equals(options.Storage.Driver, "mongo", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException("Only Mongo storage driver is supported (storage.driver == 'mongo').");
}
if (string.IsNullOrWhiteSpace(options.Storage.Dsn))
{
throw new InvalidOperationException("Storage DSN must be configured.");
}
if (options.Storage.CommandTimeoutSeconds <= 0)
{
throw new InvalidOperationException("Command timeout must be greater than zero seconds.");
}
namespace StellaOps.Concelier.WebService.Options;
public static class ConcelierOptionsValidator
{
public static void Validate(ConcelierOptions options)
{
ArgumentNullException.ThrowIfNull(options);
ValidatePostgres(options);
options.Telemetry ??= new ConcelierOptions.TelemetryOptions();
options.Authority ??= new ConcelierOptions.AuthorityOptions();
@@ -107,25 +94,25 @@ public static class ConcelierOptionsValidator
}
}
}
if (!Enum.TryParse(options.Telemetry.MinimumLogLevel, ignoreCase: true, out LogLevel _))
{
throw new InvalidOperationException($"Telemetry minimum log level '{options.Telemetry.MinimumLogLevel}' is invalid.");
}
if (!string.IsNullOrWhiteSpace(options.Telemetry.OtlpEndpoint) && !Uri.TryCreate(options.Telemetry.OtlpEndpoint, UriKind.Absolute, out _))
{
throw new InvalidOperationException("Telemetry OTLP endpoint must be an absolute URI.");
}
foreach (var attribute in options.Telemetry.ResourceAttributes)
{
if (string.IsNullOrWhiteSpace(attribute.Key))
{
throw new InvalidOperationException("Telemetry resource attribute keys must be non-empty.");
}
}
if (!Enum.TryParse(options.Telemetry.MinimumLogLevel, ignoreCase: true, out LogLevel _))
{
throw new InvalidOperationException($"Telemetry minimum log level '{options.Telemetry.MinimumLogLevel}' is invalid.");
}
if (!string.IsNullOrWhiteSpace(options.Telemetry.OtlpEndpoint) && !Uri.TryCreate(options.Telemetry.OtlpEndpoint, UriKind.Absolute, out _))
{
throw new InvalidOperationException("Telemetry OTLP endpoint must be an absolute URI.");
}
foreach (var attribute in options.Telemetry.ResourceAttributes)
{
if (string.IsNullOrWhiteSpace(attribute.Key))
{
throw new InvalidOperationException("Telemetry resource attribute keys must be non-empty.");
}
}
foreach (var header in options.Telemetry.OtlpHeaders)
{
if (string.IsNullOrWhiteSpace(header.Key))
@@ -333,4 +320,50 @@ public static class ConcelierOptionsValidator
throw new InvalidOperationException("Evidence bundle pipelineVersion must be provided.");
}
}
private static void ValidatePostgres(ConcelierOptions options)
{
var postgres = options.PostgresStorage ?? new ConcelierOptions.PostgresStorageOptions();
options.PostgresStorage = postgres;
if (!postgres.Enabled)
{
throw new InvalidOperationException("PostgreSQL storage must be enabled (postgresStorage.enabled).");
}
if (string.IsNullOrWhiteSpace(postgres.ConnectionString))
{
throw new InvalidOperationException("PostgreSQL connectionString must be configured (postgresStorage.connectionString).");
}
if (postgres.CommandTimeoutSeconds <= 0)
{
throw new InvalidOperationException("PostgreSQL commandTimeoutSeconds must be greater than zero.");
}
if (postgres.MaxPoolSize < 1)
{
throw new InvalidOperationException("PostgreSQL maxPoolSize must be greater than zero.");
}
if (postgres.MinPoolSize < 0 || postgres.MinPoolSize > postgres.MaxPoolSize)
{
throw new InvalidOperationException("PostgreSQL minPoolSize must be between 0 and maxPoolSize.");
}
if (postgres.ConnectionIdleLifetimeSeconds < 0)
{
throw new InvalidOperationException("PostgreSQL connectionIdleLifetimeSeconds must be zero or greater.");
}
if (postgres.AutoMigrate && string.IsNullOrWhiteSpace(postgres.MigrationsPath))
{
throw new InvalidOperationException("PostgreSQL migrationsPath must be configured when autoMigrate is enabled.");
}
if (string.IsNullOrWhiteSpace(postgres.SchemaName))
{
postgres.SchemaName = "vuln";
}
}
}

193
src/Concelier/StellaOps.Concelier.WebService/Program.cs
View File

@@ -26,6 +26,7 @@ using StellaOps.Concelier.Core.Events;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Core.Observations;
using StellaOps.Concelier.Core.Linksets;
using StellaOps.Concelier.Core.Diagnostics;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.WebService.Diagnostics;
using ServiceStatus = StellaOps.Concelier.WebService.Diagnostics.ServiceStatus;
@@ -54,9 +55,6 @@ using StellaOps.Concelier.Core.Aoc;
using StellaOps.Concelier.Core.Raw;
using StellaOps.Concelier.RawModels;
using StellaOps.Concelier.Storage.Postgres;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Aliases;
using StellaOps.Concelier.Core.Attestation;
using StellaOps.Concelier.Core.Signals;
using AttestationClaims = StellaOps.Concelier.Core.Attestation.AttestationClaims;
@@ -64,8 +62,10 @@ using StellaOps.Concelier.Core.Orchestration;
using System.Diagnostics.Metrics;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Aoc.AspNetCore.Results;
using StellaOps.Provenance.Mongo;
using HttpResults = Microsoft.AspNetCore.Http.Results;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Aliases;
using StellaOps.Provenance.Mongo;
namespace StellaOps.Concelier.WebService
{
@@ -91,9 +91,10 @@ builder.Host.ConfigureAppConfiguration((context, cfg) =>
{
cfg.AddInMemoryCollection(new Dictionary<string, string?>
{
{"Concelier:Storage:Dsn", Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN") ?? "mongodb://localhost:27017/test-health"},
{"Concelier:Storage:Driver", "mongo"},
{"Concelier:Storage:CommandTimeoutSeconds", "30"},
{"Concelier:PostgresStorage:Enabled", "true"},
{"Concelier:PostgresStorage:ConnectionString", Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN") ?? "Host=localhost;Port=5432;Database=concelier_test;Username=postgres;Password=postgres"},
{"Concelier:PostgresStorage:CommandTimeoutSeconds", "30"},
{"Concelier:PostgresStorage:SchemaName", "vuln"},
{"Concelier:Telemetry:Enabled", "false"}
});
}
@@ -125,11 +126,12 @@ if (builder.Environment.IsEnvironment("Testing"))
#pragma warning restore ASP0000
concelierOptions = tempProvider.GetService<IOptions<ConcelierOptions>>()?.Value ?? new ConcelierOptions
{
Storage = new ConcelierOptions.StorageOptions
PostgresStorage = new ConcelierOptions.PostgresStorageOptions
{
Dsn = Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN") ?? "mongodb://localhost:27017/test-health",
Driver = "mongo",
CommandTimeoutSeconds = 30
Enabled = true,
ConnectionString = Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN") ?? "Host=localhost;Port=5432;Database=concelier_test;Username=postgres;Password=postgres",
CommandTimeoutSeconds = 30,
SchemaName = "vuln"
},
Telemetry = new ConcelierOptions.TelemetryOptions
{
@@ -137,10 +139,18 @@ if (builder.Environment.IsEnvironment("Testing"))
}
};
concelierOptions.Storage ??= new ConcelierOptions.StorageOptions();
concelierOptions.Storage.Dsn = Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN") ?? "mongodb://localhost:27017/orch-tests";
concelierOptions.Storage.Driver = "mongo";
concelierOptions.Storage.CommandTimeoutSeconds = concelierOptions.Storage.CommandTimeoutSeconds <= 0 ? 30 : concelierOptions.Storage.CommandTimeoutSeconds;
concelierOptions.PostgresStorage ??= new ConcelierOptions.PostgresStorageOptions
{
Enabled = true,
ConnectionString = Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN") ?? "Host=localhost;Port=5432;Database=concelier_test;Username=postgres;Password=postgres",
CommandTimeoutSeconds = 30,
SchemaName = "vuln"
};
if (string.IsNullOrWhiteSpace(concelierOptions.PostgresStorage.ConnectionString))
{
concelierOptions.PostgresStorage.ConnectionString = Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN") ?? string.Empty;
}
ConcelierOptionsPostConfigure.Apply(concelierOptions, contentRootPath);
// Skip validation in Testing to allow factory-provided wiring.
@@ -149,10 +159,21 @@ else
{
concelierOptions = builder.Configuration.BindOptions<ConcelierOptions>(postConfigure: (opts, _) =>
{
var testDsn = Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN");
if (string.IsNullOrWhiteSpace(opts.Storage.Dsn) && !string.IsNullOrWhiteSpace(testDsn))
var testDsn = Environment.GetEnvironmentVariable("CONCELIER_POSTGRES_DSN")
?? Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN");
opts.PostgresStorage ??= new ConcelierOptions.PostgresStorageOptions
{
opts.Storage.Dsn = testDsn;
Enabled = !string.IsNullOrWhiteSpace(testDsn),
ConnectionString = testDsn ?? string.Empty,
SchemaName = "vuln",
CommandTimeoutSeconds = 30
};
if (string.IsNullOrWhiteSpace(opts.PostgresStorage.ConnectionString) && !string.IsNullOrWhiteSpace(testDsn))
{
opts.PostgresStorage.ConnectionString = testDsn;
opts.PostgresStorage.Enabled = true;
}
ConcelierOptionsPostConfigure.Apply(opts, contentRootPath);
@@ -179,24 +200,26 @@ builder.Services.AddSingleton<MirrorFileLocator>();
var isTesting = builder.Environment.IsEnvironment("Testing");
// Add PostgreSQL storage for LNM linkset cache if configured.
// This provides a PostgreSQL-backed implementation of IAdvisoryLinksetStore for the read-through cache.
if (concelierOptions.PostgresStorage is { Enabled: true } postgresOptions)
// Add PostgreSQL storage for all Concelier persistence.
var postgresOptions = concelierOptions.PostgresStorage ?? throw new InvalidOperationException("PostgreSQL storage must be configured.");
if (!postgresOptions.Enabled)
{
builder.Services.AddConcelierPostgresStorage(pgOptions =>
{
pgOptions.ConnectionString = postgresOptions.ConnectionString;
pgOptions.CommandTimeoutSeconds = postgresOptions.CommandTimeoutSeconds;
pgOptions.MaxPoolSize = postgresOptions.MaxPoolSize;
pgOptions.MinPoolSize = postgresOptions.MinPoolSize;
pgOptions.ConnectionIdleLifetimeSeconds = postgresOptions.ConnectionIdleLifetimeSeconds;
pgOptions.Pooling = postgresOptions.Pooling;
pgOptions.SchemaName = postgresOptions.SchemaName;
pgOptions.AutoMigrate = postgresOptions.AutoMigrate;
pgOptions.MigrationsPath = postgresOptions.MigrationsPath;
});
throw new InvalidOperationException("PostgreSQL storage must be enabled.");
}
builder.Services.AddConcelierPostgresStorage(pgOptions =>
{
pgOptions.ConnectionString = postgresOptions.ConnectionString;
pgOptions.CommandTimeoutSeconds = postgresOptions.CommandTimeoutSeconds;
pgOptions.MaxPoolSize = postgresOptions.MaxPoolSize;
pgOptions.MinPoolSize = postgresOptions.MinPoolSize;
pgOptions.ConnectionIdleLifetimeSeconds = postgresOptions.ConnectionIdleLifetimeSeconds;
pgOptions.Pooling = postgresOptions.Pooling;
pgOptions.SchemaName = postgresOptions.SchemaName;
pgOptions.AutoMigrate = postgresOptions.AutoMigrate;
pgOptions.MigrationsPath = postgresOptions.MigrationsPath;
});
builder.Services.AddOptions<AdvisoryObservationEventPublisherOptions>()
.Bind(builder.Configuration.GetSection("advisoryObservationEvents"))
.PostConfigure(options =>
@@ -1039,9 +1062,12 @@ var advisoryIngestEndpoint = app.MapPost("/ingest/advisory", async (
return Problem(context, "Invalid advisory payload", StatusCodes.Status400BadRequest, ProblemTypes.Validation, ex.Message);
}
var chunkStopwatch = Stopwatch.StartNew();
try
{
var result = await rawService.IngestAsync(document, cancellationToken).ConfigureAwait(false);
chunkStopwatch.Stop();
var response = new AdvisoryIngestResponse(
result.Record.Id,
@@ -1065,10 +1091,21 @@ var advisoryIngestEndpoint = app.MapPost("/ingest/advisory", async (
ingestRequest.Source.Vendor ?? "(unknown)",
result.Inserted ? "inserted" : "duplicate"));
var telemetrySource = ingestRequest.Source.Vendor ?? "(unknown)";
var (_, _, conflicts) = AdvisoryLinksetNormalization.FromRawLinksetWithConfidence(document.Linkset, providedConfidence: null);
var collisionCount = VulnExplorerTelemetry.CountAliasCollisions(conflicts);
VulnExplorerTelemetry.RecordIdentifierCollisions(tenant, telemetrySource, collisionCount);
VulnExplorerTelemetry.RecordChunkLatency(tenant, telemetrySource, chunkStopwatch.Elapsed);
if (VulnExplorerTelemetry.IsWithdrawn(document.Content.Raw))
{
VulnExplorerTelemetry.RecordWithdrawnStatement(tenant, telemetrySource);
}
return JsonResult(response, statusCode);
}
catch (ConcelierAocGuardException guardException)
{
chunkStopwatch.Stop();
logger.LogWarning(
guardException,
"AOC guard rejected advisory ingest tenant={Tenant} upstream={UpstreamId} requestHash={RequestHash} documentHash={DocumentHash} codes={Codes}",
@@ -2115,6 +2152,12 @@ var advisoryChunksEndpoint = app.MapGet("/advisories/{advisoryKey}/chunks", asyn
buildResult.Response.Entries.Count,
duration,
guardrailCounts));
VulnExplorerTelemetry.RecordChunkRequest(
tenant!,
result: "ok",
cacheHit,
buildResult.Response.Entries.Count,
duration.TotalMilliseconds);
return JsonResult(buildResult.Response);
});
@@ -3269,7 +3312,7 @@ void ApplyNoCache(HttpResponse response)
response.Headers["Expires"] = "0";
}
await InitializeMongoAsync(app);
await InitializePostgresAsync(app);
app.MapGet("/health", ([FromServices] IOptions<ConcelierOptions> opts, [FromServices] StellaOps.Concelier.WebService.Diagnostics.ServiceStatus status, HttpContext context) =>
{
@@ -3278,11 +3321,12 @@ app.MapGet("/health", ([FromServices] IOptions<ConcelierOptions> opts, [FromServ
var snapshot = status.CreateSnapshot();
var uptimeSeconds = Math.Max((snapshot.CapturedAt - snapshot.StartedAt).TotalSeconds, 0d);
var storage = new StorageBootstrapHealth(
Driver: opts.Value.Storage.Driver,
Completed: snapshot.BootstrapCompletedAt is not null,
CompletedAt: snapshot.BootstrapCompletedAt,
DurationMs: snapshot.BootstrapDuration?.TotalMilliseconds);
var storage = new StorageHealth(
Backend: "postgres",
Ready: snapshot.LastReadySucceeded,
CheckedAt: snapshot.LastReadyCheckAt,
LatencyMs: snapshot.LastStorageLatency?.TotalMilliseconds,
Error: snapshot.LastStorageError);
var telemetry = new TelemetryHealth(
Enabled: opts.Value.Telemetry.Enabled,
@@ -3300,24 +3344,32 @@ app.MapGet("/health", ([FromServices] IOptions<ConcelierOptions> opts, [FromServ
return JsonResult(response);
});
app.MapGet("/ready", ([FromServices] StellaOps.Concelier.WebService.Diagnostics.ServiceStatus status, HttpContext context) =>
app.MapGet("/ready", async (
[FromServices] StellaOps.Concelier.WebService.Diagnostics.ServiceStatus status,
[FromServices] ConcelierDataSource dataSource,
HttpContext context,
CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
var (ready, latency, error) = await CheckPostgresAsync(dataSource, cancellationToken).ConfigureAwait(false);
status.RecordStorageCheck(ready, latency, error);
var snapshot = status.CreateSnapshot();
var uptimeSeconds = Math.Max((snapshot.CapturedAt - snapshot.StartedAt).TotalSeconds, 0d);
var mongo = new MongoReadyHealth(
Status: "bypassed",
LatencyMs: null,
var storage = new StorageHealth(
Backend: "postgres",
Ready: ready,
CheckedAt: snapshot.LastReadyCheckAt,
Error: "mongo disabled");
LatencyMs: latency.TotalMilliseconds,
Error: error);
var response = new ReadyDocument(
Status: "ready",
Status: ready ? "ready" : "degraded",
StartedAt: snapshot.StartedAt,
UptimeSeconds: uptimeSeconds,
Mongo: mongo);
Storage: storage);
return JsonResult(response);
});
@@ -4019,9 +4071,54 @@ static SignalsSymbolSetResponse ToSymbolSetResponse(AffectedSymbolSet symbolSet)
return pluginOptions;
}
static async Task InitializeMongoAsync(WebApplication app)
static async Task InitializePostgresAsync(WebApplication app)
{
await Task.CompletedTask;
var dataSource = app.Services.GetService<ConcelierDataSource>();
var status = app.Services.GetRequiredService<StellaOps.Concelier.WebService.Diagnostics.ServiceStatus>();
if (dataSource is null)
{
status.RecordStorageCheck(false, TimeSpan.Zero, "PostgreSQL storage not configured");
return;
}
var stopwatch = Stopwatch.StartNew();
try
{
var (ready, latency, error) = await CheckPostgresAsync(dataSource, CancellationToken.None).ConfigureAwait(false);
stopwatch.Stop();
status.RecordStorageCheck(ready, latency, error);
if (ready)
{
status.MarkBootstrapCompleted(latency);
}
}
catch (Exception ex)
{
stopwatch.Stop();
status.RecordStorageCheck(false, stopwatch.Elapsed, ex.Message);
}
}
static async Task<(bool Ready, TimeSpan Latency, string? Error)> CheckPostgresAsync(
ConcelierDataSource dataSource,
CancellationToken cancellationToken)
{
var stopwatch = Stopwatch.StartNew();
try
{
await using var connection = await dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var command = connection.CreateCommand();
command.CommandText = "select 1";
_ = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
stopwatch.Stop();
return (true, stopwatch.Elapsed, null);
}
catch (Exception ex)
{
stopwatch.Stop();
return (false, stopwatch.Elapsed, ex.Message);
}
}
}

2
src/Concelier/StellaOps.Concelier.WebService/StellaOps.Concelier.WebService.csproj
View File

@@ -41,4 +41,4 @@
OutputItemType="Analyzer"
ReferenceOutputAssembly="false" />
</ItemGroup>
</Project>
</Project>

112
src/Concelier/StellaOps.Concelier.sln
View File

@@ -185,6 +185,22 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Analyze
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Ingestion.Telemetry", "..\__Libraries\StellaOps.Ingestion.Telemetry\StellaOps.Ingestion.Telemetry.csproj", "{85D215EC-DCFE-4F7F-BB07-540DCF66BE8C}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cryptography.Plugin.SmRemote", "..\__Libraries\StellaOps.Cryptography.Plugin.SmRemote\StellaOps.Cryptography.Plugin.SmRemote.csproj", "{FCA91451-5D4A-4E75-9268-B253A902A726}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.SmRemote.Service", "..\SmRemote\StellaOps.SmRemote.Service\StellaOps.SmRemote.Service.csproj", "{E823EB56-86F4-4989-9480-9F1D8DD780F8}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cryptography.Plugin.SmSoft", "..\__Libraries\StellaOps.Cryptography.Plugin.SmSoft\StellaOps.Cryptography.Plugin.SmSoft.csproj", "{64C7E443-CD2C-475E-B9C6-95EF8160F4D8}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cryptography.DependencyInjection", "..\__Libraries\StellaOps.Cryptography.DependencyInjection\StellaOps.Cryptography.DependencyInjection.csproj", "{1A7ACB4E-FDCD-4AA9-8516-EC60D8A25922}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cryptography.Plugin.Pkcs11Gost", "..\__Libraries\StellaOps.Cryptography.Plugin.Pkcs11Gost\StellaOps.Cryptography.Plugin.Pkcs11Gost.csproj", "{3CC87BD4-38B7-421B-9688-B2ED2B392646}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cryptography.Plugin.OpenSslGost", "..\__Libraries\StellaOps.Cryptography.Plugin.OpenSslGost\StellaOps.Cryptography.Plugin.OpenSslGost.csproj", "{27052CD3-98B4-4D37-88F9-7D8B54363F74}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cryptography.Plugin.PqSoft", "..\__Libraries\StellaOps.Cryptography.Plugin.PqSoft\StellaOps.Cryptography.Plugin.PqSoft.csproj", "{29B6BB6D-A002-41A6-B3F9-F6F894F2A8D2}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cryptography.Plugin.WineCsp", "..\__Libraries\StellaOps.Cryptography.Plugin.WineCsp\StellaOps.Cryptography.Plugin.WineCsp.csproj", "{98908D4F-1A48-4CED-B2CF-92C3179B44FD}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -1251,6 +1267,102 @@ Global
{85D215EC-DCFE-4F7F-BB07-540DCF66BE8C}.Release|x64.Build.0 = Release|Any CPU
{85D215EC-DCFE-4F7F-BB07-540DCF66BE8C}.Release|x86.ActiveCfg = Release|Any CPU
{85D215EC-DCFE-4F7F-BB07-540DCF66BE8C}.Release|x86.Build.0 = Release|Any CPU
{FCA91451-5D4A-4E75-9268-B253A902A726}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{FCA91451-5D4A-4E75-9268-B253A902A726}.Debug|Any CPU.Build.0 = Debug|Any CPU
{FCA91451-5D4A-4E75-9268-B253A902A726}.Debug|x64.ActiveCfg = Debug|Any CPU
{FCA91451-5D4A-4E75-9268-B253A902A726}.Debug|x64.Build.0 = Debug|Any CPU
{FCA91451-5D4A-4E75-9268-B253A902A726}.Debug|x86.ActiveCfg = Debug|Any CPU
{FCA91451-5D4A-4E75-9268-B253A902A726}.Debug|x86.Build.0 = Debug|Any CPU
{FCA91451-5D4A-4E75-9268-B253A902A726}.Release|Any CPU.ActiveCfg = Release|Any CPU
{FCA91451-5D4A-4E75-9268-B253A902A726}.Release|Any CPU.Build.0 = Release|Any CPU
{FCA91451-5D4A-4E75-9268-B253A902A726}.Release|x64.ActiveCfg = Release|Any CPU
{FCA91451-5D4A-4E75-9268-B253A902A726}.Release|x64.Build.0 = Release|Any CPU
{FCA91451-5D4A-4E75-9268-B253A902A726}.Release|x86.ActiveCfg = Release|Any CPU
{FCA91451-5D4A-4E75-9268-B253A902A726}.Release|x86.Build.0 = Release|Any CPU
{E823EB56-86F4-4989-9480-9F1D8DD780F8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E823EB56-86F4-4989-9480-9F1D8DD780F8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E823EB56-86F4-4989-9480-9F1D8DD780F8}.Debug|x64.ActiveCfg = Debug|Any CPU
{E823EB56-86F4-4989-9480-9F1D8DD780F8}.Debug|x64.Build.0 = Debug|Any CPU
{E823EB56-86F4-4989-9480-9F1D8DD780F8}.Debug|x86.ActiveCfg = Debug|Any CPU
{E823EB56-86F4-4989-9480-9F1D8DD780F8}.Debug|x86.Build.0 = Debug|Any CPU
{E823EB56-86F4-4989-9480-9F1D8DD780F8}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E823EB56-86F4-4989-9480-9F1D8DD780F8}.Release|Any CPU.Build.0 = Release|Any CPU
{E823EB56-86F4-4989-9480-9F1D8DD780F8}.Release|x64.ActiveCfg = Release|Any CPU
{E823EB56-86F4-4989-9480-9F1D8DD780F8}.Release|x64.Build.0 = Release|Any CPU
{E823EB56-86F4-4989-9480-9F1D8DD780F8}.Release|x86.ActiveCfg = Release|Any CPU
{E823EB56-86F4-4989-9480-9F1D8DD780F8}.Release|x86.Build.0 = Release|Any CPU
{64C7E443-CD2C-475E-B9C6-95EF8160F4D8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{64C7E443-CD2C-475E-B9C6-95EF8160F4D8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{64C7E443-CD2C-475E-B9C6-95EF8160F4D8}.Debug|x64.ActiveCfg = Debug|Any CPU
{64C7E443-CD2C-475E-B9C6-95EF8160F4D8}.Debug|x64.Build.0 = Debug|Any CPU
{64C7E443-CD2C-475E-B9C6-95EF8160F4D8}.Debug|x86.ActiveCfg = Debug|Any CPU
{64C7E443-CD2C-475E-B9C6-95EF8160F4D8}.Debug|x86.Build.0 = Debug|Any CPU
{64C7E443-CD2C-475E-B9C6-95EF8160F4D8}.Release|Any CPU.ActiveCfg = Release|Any CPU
{64C7E443-CD2C-475E-B9C6-95EF8160F4D8}.Release|Any CPU.Build.0 = Release|Any CPU
{64C7E443-CD2C-475E-B9C6-95EF8160F4D8}.Release|x64.ActiveCfg = Release|Any CPU
{64C7E443-CD2C-475E-B9C6-95EF8160F4D8}.Release|x64.Build.0 = Release|Any CPU
{64C7E443-CD2C-475E-B9C6-95EF8160F4D8}.Release|x86.ActiveCfg = Release|Any CPU
{64C7E443-CD2C-475E-B9C6-95EF8160F4D8}.Release|x86.Build.0 = Release|Any CPU
{1A7ACB4E-FDCD-4AA9-8516-EC60D8A25922}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{1A7ACB4E-FDCD-4AA9-8516-EC60D8A25922}.Debug|Any CPU.Build.0 = Debug|Any CPU
{1A7ACB4E-FDCD-4AA9-8516-EC60D8A25922}.Debug|x64.ActiveCfg = Debug|Any CPU
{1A7ACB4E-FDCD-4AA9-8516-EC60D8A25922}.Debug|x64.Build.0 = Debug|Any CPU
{1A7ACB4E-FDCD-4AA9-8516-EC60D8A25922}.Debug|x86.ActiveCfg = Debug|Any CPU
{1A7ACB4E-FDCD-4AA9-8516-EC60D8A25922}.Debug|x86.Build.0 = Debug|Any CPU
{1A7ACB4E-FDCD-4AA9-8516-EC60D8A25922}.Release|Any CPU.ActiveCfg = Release|Any CPU
{1A7ACB4E-FDCD-4AA9-8516-EC60D8A25922}.Release|Any CPU.Build.0 = Release|Any CPU
{1A7ACB4E-FDCD-4AA9-8516-EC60D8A25922}.Release|x64.ActiveCfg = Release|Any CPU
{1A7ACB4E-FDCD-4AA9-8516-EC60D8A25922}.Release|x64.Build.0 = Release|Any CPU
{1A7ACB4E-FDCD-4AA9-8516-EC60D8A25922}.Release|x86.ActiveCfg = Release|Any CPU
{1A7ACB4E-FDCD-4AA9-8516-EC60D8A25922}.Release|x86.Build.0 = Release|Any CPU
{3CC87BD4-38B7-421B-9688-B2ED2B392646}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3CC87BD4-38B7-421B-9688-B2ED2B392646}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3CC87BD4-38B7-421B-9688-B2ED2B392646}.Debug|x64.ActiveCfg = Debug|Any CPU
{3CC87BD4-38B7-421B-9688-B2ED2B392646}.Debug|x64.Build.0 = Debug|Any CPU
{3CC87BD4-38B7-421B-9688-B2ED2B392646}.Debug|x86.ActiveCfg = Debug|Any CPU
{3CC87BD4-38B7-421B-9688-B2ED2B392646}.Debug|x86.Build.0 = Debug|Any CPU
{3CC87BD4-38B7-421B-9688-B2ED2B392646}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3CC87BD4-38B7-421B-9688-B2ED2B392646}.Release|Any CPU.Build.0 = Release|Any CPU
{3CC87BD4-38B7-421B-9688-B2ED2B392646}.Release|x64.ActiveCfg = Release|Any CPU
{3CC87BD4-38B7-421B-9688-B2ED2B392646}.Release|x64.Build.0 = Release|Any CPU
{3CC87BD4-38B7-421B-9688-B2ED2B392646}.Release|x86.ActiveCfg = Release|Any CPU
{3CC87BD4-38B7-421B-9688-B2ED2B392646}.Release|x86.Build.0 = Release|Any CPU
{27052CD3-98B4-4D37-88F9-7D8B54363F74}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{27052CD3-98B4-4D37-88F9-7D8B54363F74}.Debug|Any CPU.Build.0 = Debug|Any CPU
{27052CD3-98B4-4D37-88F9-7D8B54363F74}.Debug|x64.ActiveCfg = Debug|Any CPU
{27052CD3-98B4-4D37-88F9-7D8B54363F74}.Debug|x64.Build.0 = Debug|Any CPU
{27052CD3-98B4-4D37-88F9-7D8B54363F74}.Debug|x86.ActiveCfg = Debug|Any CPU
{27052CD3-98B4-4D37-88F9-7D8B54363F74}.Debug|x86.Build.0 = Debug|Any CPU
{27052CD3-98B4-4D37-88F9-7D8B54363F74}.Release|Any CPU.ActiveCfg = Release|Any CPU
{27052CD3-98B4-4D37-88F9-7D8B54363F74}.Release|Any CPU.Build.0 = Release|Any CPU
{27052CD3-98B4-4D37-88F9-7D8B54363F74}.Release|x64.ActiveCfg = Release|Any CPU
{27052CD3-98B4-4D37-88F9-7D8B54363F74}.Release|x64.Build.0 = Release|Any CPU
{27052CD3-98B4-4D37-88F9-7D8B54363F74}.Release|x86.ActiveCfg = Release|Any CPU
{27052CD3-98B4-4D37-88F9-7D8B54363F74}.Release|x86.Build.0 = Release|Any CPU
{29B6BB6D-A002-41A6-B3F9-F6F894F2A8D2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{29B6BB6D-A002-41A6-B3F9-F6F894F2A8D2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{29B6BB6D-A002-41A6-B3F9-F6F894F2A8D2}.Debug|x64.ActiveCfg = Debug|Any CPU
{29B6BB6D-A002-41A6-B3F9-F6F894F2A8D2}.Debug|x64.Build.0 = Debug|Any CPU
{29B6BB6D-A002-41A6-B3F9-F6F894F2A8D2}.Debug|x86.ActiveCfg = Debug|Any CPU
{29B6BB6D-A002-41A6-B3F9-F6F894F2A8D2}.Debug|x86.Build.0 = Debug|Any CPU
{29B6BB6D-A002-41A6-B3F9-F6F894F2A8D2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{29B6BB6D-A002-41A6-B3F9-F6F894F2A8D2}.Release|Any CPU.Build.0 = Release|Any CPU
{29B6BB6D-A002-41A6-B3F9-F6F894F2A8D2}.Release|x64.ActiveCfg = Release|Any CPU
{29B6BB6D-A002-41A6-B3F9-F6F894F2A8D2}.Release|x64.Build.0 = Release|Any CPU
{29B6BB6D-A002-41A6-B3F9-F6F894F2A8D2}.Release|x86.ActiveCfg = Release|Any CPU
{29B6BB6D-A002-41A6-B3F9-F6F894F2A8D2}.Release|x86.Build.0 = Release|Any CPU
{98908D4F-1A48-4CED-B2CF-92C3179B44FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{98908D4F-1A48-4CED-B2CF-92C3179B44FD}.Debug|Any CPU.Build.0 = Debug|Any CPU
{98908D4F-1A48-4CED-B2CF-92C3179B44FD}.Debug|x64.ActiveCfg = Debug|Any CPU
{98908D4F-1A48-4CED-B2CF-92C3179B44FD}.Debug|x64.Build.0 = Debug|Any CPU
{98908D4F-1A48-4CED-B2CF-92C3179B44FD}.Debug|x86.ActiveCfg = Debug|Any CPU
{98908D4F-1A48-4CED-B2CF-92C3179B44FD}.Debug|x86.Build.0 = Debug|Any CPU
{98908D4F-1A48-4CED-B2CF-92C3179B44FD}.Release|Any CPU.ActiveCfg = Release|Any CPU
{98908D4F-1A48-4CED-B2CF-92C3179B44FD}.Release|Any CPU.Build.0 = Release|Any CPU
{98908D4F-1A48-4CED-B2CF-92C3179B44FD}.Release|x64.ActiveCfg = Release|Any CPU
{98908D4F-1A48-4CED-B2CF-92C3179B44FD}.Release|x64.Build.0 = Release|Any CPU
{98908D4F-1A48-4CED-B2CF-92C3179B44FD}.Release|x86.ActiveCfg = Release|Any CPU
{98908D4F-1A48-4CED-B2CF-92C3179B44FD}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

143
src/Concelier/__Libraries/StellaOps.Concelier.Core/Diagnostics/VulnExplorerTelemetry.cs Normal file
View File

@@ -0,0 +1,143 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
using System.Linq;
using System.Text.Json;
using StellaOps.Concelier.Core.Linksets;
namespace StellaOps.Concelier.Core.Diagnostics;
/// <summary>
/// Metrics exported for Vuln Explorer consumers (fact-only telemetry).
/// </summary>
public static class VulnExplorerTelemetry
{
public const string MeterName = "StellaOps.Concelier.VulnExplorer";
private static readonly Meter Meter = new(MeterName);
private static readonly Counter<long> IdentifierCollisionCounter = Meter.CreateCounter<long>(
"vuln.identifier_collisions_total",
unit: "collision",
description: "Identifier/alias collisions detected while aggregating linksets for Vuln Explorer.");
private static readonly Counter<long> WithdrawnStatementCounter = Meter.CreateCounter<long>(
"vuln.withdrawn_statements_total",
unit: "statement",
description: "Withdrawn advisory observations detected by change emitters.");
private static readonly Counter<long> ChunkRequestCounter = Meter.CreateCounter<long>(
"vuln.chunk_requests_total",
unit: "request",
description: "Advisory chunk requests served for Vuln Explorer evidence panels.");
private static readonly Histogram<double> ChunkLatencyHistogram = Meter.CreateHistogram<double>(
"vuln.chunk_latency_ms",
unit: "ms",
description: "Latency to build advisory chunks (fact-only) for Vuln Explorer.");
public static void RecordIdentifierCollisions(string tenant, string? source, int collisions)
{
if (collisions <= 0 || string.IsNullOrWhiteSpace(tenant))
{
return;
}
var tags = new[]
{
KeyValuePair.Create<string, object?>("tenant", tenant),
KeyValuePair.Create<string, object?>("source", source ?? "unknown")
};
IdentifierCollisionCounter.Add(collisions, tags);
}
public static int CountAliasCollisions(IReadOnlyList<AdvisoryLinksetConflict>? conflicts)
{
if (conflicts is null || conflicts.Count == 0)
{
return 0;
}
return conflicts.Count(conflict =>
string.Equals(conflict.Reason, "alias-inconsistency", StringComparison.OrdinalIgnoreCase) ||
string.Equals(conflict.Field, "aliases", StringComparison.OrdinalIgnoreCase));
}
public static void RecordWithdrawnStatement(string tenant, string? source)
{
if (string.IsNullOrWhiteSpace(tenant))
{
return;
}
var tags = new[]
{
KeyValuePair.Create<string, object?>("tenant", tenant),
KeyValuePair.Create<string, object?>("source", source ?? "unknown")
};
WithdrawnStatementCounter.Add(1, tags);
}
public static void RecordChunkRequest(string tenant, string result, bool cacheHit, int chunkCount, double latencyMs)
{
if (string.IsNullOrWhiteSpace(tenant))
{
return;
}
var sanitizedResult = string.IsNullOrWhiteSpace(result) ? "unknown" : result.Trim().ToLowerInvariant();
var safeLatency = latencyMs < 0 ? 0d : latencyMs;
var normalizedChunkCount = chunkCount < 0 ? 0 : chunkCount;
var tags = new[]
{
KeyValuePair.Create<string, object?>("tenant", tenant),
KeyValuePair.Create<string, object?>("result", sanitizedResult),
KeyValuePair.Create<string, object?>("cache_hit", cacheHit),
KeyValuePair.Create<string, object?>("chunk_count", normalizedChunkCount)
};
ChunkRequestCounter.Add(1, tags);
ChunkLatencyHistogram.Record(safeLatency, tags);
}
public static void RecordChunkLatency(string tenant, string? source, TimeSpan duration)
{
if (string.IsNullOrWhiteSpace(tenant))
{
return;
}
var tags = new[]
{
KeyValuePair.Create<string, object?>("tenant", tenant),
KeyValuePair.Create<string, object?>("source", source ?? "unknown")
};
ChunkLatencyHistogram.Record(Math.Max(0, duration.TotalMilliseconds), tags);
}
public static bool IsWithdrawn(JsonElement content)
{
if (content.ValueKind != JsonValueKind.Object)
{
return false;
}
if (content.TryGetProperty("withdrawn", out var withdrawnElement) &&
withdrawnElement.ValueKind == JsonValueKind.True)
{
return true;
}
if (content.TryGetProperty("withdrawn_at", out var withdrawnAtElement) &&
withdrawnAtElement.ValueKind is JsonValueKind.String)
{
return !string.IsNullOrWhiteSpace(withdrawnAtElement.GetString());
}
return false;
}
}

2
src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets/AdvisoryLinksetNormalization.cs
View File

@@ -7,7 +7,7 @@ using StellaOps.Concelier.Normalization.SemVer;
namespace StellaOps.Concelier.Core.Linksets;
internal static class AdvisoryLinksetNormalization
public static class AdvisoryLinksetNormalization
{
public static AdvisoryLinksetNormalized? FromRawLinkset(RawLinkset linkset)
{

386
src/Concelier/__Libraries/StellaOps.Concelier.Core/Observations/AdvisoryObservationQueryService.cs
View File

@@ -5,192 +5,194 @@ using StellaOps.Concelier.Models;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.RawModels;
using StellaOps.Concelier.Core.Linksets;
namespace StellaOps.Concelier.Core.Observations;
/// <summary>
/// Default implementation of <see cref="IAdvisoryObservationQueryService"/> that projects raw observations for overlay consumers.
/// </summary>
public sealed class AdvisoryObservationQueryService : IAdvisoryObservationQueryService
{
private const int DefaultPageSize = 200;
private const int MaxPageSize = 500;
private readonly IAdvisoryObservationLookup _lookup;
public AdvisoryObservationQueryService(IAdvisoryObservationLookup lookup)
{
_lookup = lookup ?? throw new ArgumentNullException(nameof(lookup));
}
public async ValueTask<AdvisoryObservationQueryResult> QueryAsync(
AdvisoryObservationQueryOptions options,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(options);
cancellationToken.ThrowIfCancellationRequested();
var normalizedTenant = NormalizeTenant(options.Tenant);
var normalizedObservationIds = NormalizeSet(options.ObservationIds, static value => value, StringComparer.Ordinal);
using StellaOps.Concelier.Core.Diagnostics;
namespace StellaOps.Concelier.Core.Observations;
/// <summary>
/// Default implementation of <see cref="IAdvisoryObservationQueryService"/> that projects raw observations for overlay consumers.
/// </summary>
public sealed class AdvisoryObservationQueryService : IAdvisoryObservationQueryService
{
private const int DefaultPageSize = 200;
private const int MaxPageSize = 500;
private readonly IAdvisoryObservationLookup _lookup;
public AdvisoryObservationQueryService(IAdvisoryObservationLookup lookup)
{
_lookup = lookup ?? throw new ArgumentNullException(nameof(lookup));
}
public async ValueTask<AdvisoryObservationQueryResult> QueryAsync(
AdvisoryObservationQueryOptions options,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(options);
cancellationToken.ThrowIfCancellationRequested();
var normalizedTenant = NormalizeTenant(options.Tenant);
var normalizedObservationIds = NormalizeSet(options.ObservationIds, static value => value, StringComparer.Ordinal);
var normalizedAliases = NormalizeSet(options.Aliases, static value => value, StringComparer.OrdinalIgnoreCase);
var normalizedPurls = NormalizeSet(options.Purls, static value => value, StringComparer.Ordinal);
var normalizedCpes = NormalizeSet(options.Cpes, static value => value, StringComparer.Ordinal);
var limit = NormalizeLimit(options.Limit);
var fetchSize = checked(limit + 1);
var cursor = DecodeCursor(options.Cursor);
var observations = await _lookup
.FindByFiltersAsync(
normalizedTenant,
normalizedObservationIds,
normalizedAliases,
normalizedPurls,
normalizedCpes,
cursor,
fetchSize,
cancellationToken)
.ConfigureAwait(false);
var ordered = observations
.Where(observation => Matches(observation, normalizedObservationIds, normalizedAliases, normalizedPurls, normalizedCpes))
.OrderByDescending(static observation => observation.CreatedAt)
.ThenBy(static observation => observation.ObservationId, StringComparer.Ordinal)
.ToImmutableArray();
var hasMore = ordered.Length > limit;
var page = hasMore ? ordered.Take(limit).ToImmutableArray() : ordered;
var nextCursor = hasMore ? EncodeCursor(page[^1]) : null;
var linkset = BuildAggregateLinkset(page);
return new AdvisoryObservationQueryResult(page, linkset, nextCursor, hasMore);
}
private static bool Matches(
AdvisoryObservation observation,
ImmutableHashSet<string> observationIds,
ImmutableHashSet<string> aliases,
ImmutableHashSet<string> purls,
ImmutableHashSet<string> cpes)
{
ArgumentNullException.ThrowIfNull(observation);
if (observationIds.Count > 0 && !observationIds.Contains(observation.ObservationId))
{
return false;
}
if (aliases.Count > 0 && !observation.Linkset.Aliases.Any(aliases.Contains))
{
return false;
}
if (purls.Count > 0 && !observation.Linkset.Purls.Any(purls.Contains))
{
return false;
}
if (cpes.Count > 0 && !observation.Linkset.Cpes.Any(cpes.Contains))
{
return false;
}
return true;
}
private static string NormalizeTenant(string tenant)
=> Validation.EnsureNotNullOrWhiteSpace(tenant, nameof(tenant)).ToLowerInvariant();
private static ImmutableHashSet<string> NormalizeSet(
IEnumerable<string>? values,
Func<string, string> projector,
StringComparer comparer)
{
if (values is null)
{
return ImmutableHashSet<string>.Empty;
}
var builder = ImmutableHashSet.CreateBuilder<string>(comparer);
foreach (var value in values)
{
var normalized = Validation.TrimToNull(value);
if (normalized is null)
{
continue;
}
builder.Add(projector(normalized));
}
return builder.ToImmutable();
}
private static int NormalizeLimit(int? requestedLimit)
{
if (!requestedLimit.HasValue || requestedLimit.Value <= 0)
{
return DefaultPageSize;
}
var limit = requestedLimit.Value;
if (limit > MaxPageSize)
{
return MaxPageSize;
}
return limit;
}
private static AdvisoryObservationCursor? DecodeCursor(string? cursor)
{
if (string.IsNullOrWhiteSpace(cursor))
{
return null;
}
try
{
var decoded = Convert.FromBase64String(cursor.Trim());
var payload = Encoding.UTF8.GetString(decoded);
var separator = payload.IndexOf(':');
if (separator <= 0 || separator >= payload.Length - 1)
{
throw new FormatException("Cursor is malformed.");
}
var ticksText = payload.AsSpan(0, separator);
if (!long.TryParse(ticksText, NumberStyles.Integer, CultureInfo.InvariantCulture, out var ticks))
{
throw new FormatException("Cursor timestamp is invalid.");
}
var createdAt = new DateTimeOffset(DateTime.SpecifyKind(new DateTime(ticks), DateTimeKind.Utc));
var observationId = payload[(separator + 1)..];
if (string.IsNullOrWhiteSpace(observationId))
{
throw new FormatException("Cursor observation id is missing.");
}
return new AdvisoryObservationCursor(createdAt, observationId);
}
catch (FormatException)
{
throw;
}
catch (Exception ex)
{
throw new FormatException("Cursor is malformed.", ex);
}
}
private static string? EncodeCursor(AdvisoryObservation observation)
{
if (observation is null)
{
return null;
}
var normalizedPurls = NormalizeSet(options.Purls, static value => value, StringComparer.Ordinal);
var normalizedCpes = NormalizeSet(options.Cpes, static value => value, StringComparer.Ordinal);
var limit = NormalizeLimit(options.Limit);
var fetchSize = checked(limit + 1);
var cursor = DecodeCursor(options.Cursor);
var observations = await _lookup
.FindByFiltersAsync(
normalizedTenant,
normalizedObservationIds,
normalizedAliases,
normalizedPurls,
normalizedCpes,
cursor,
fetchSize,
cancellationToken)
.ConfigureAwait(false);
var ordered = observations
.Where(observation => Matches(observation, normalizedObservationIds, normalizedAliases, normalizedPurls, normalizedCpes))
.OrderByDescending(static observation => observation.CreatedAt)
.ThenBy(static observation => observation.ObservationId, StringComparer.Ordinal)
.ToImmutableArray();
var hasMore = ordered.Length > limit;
var page = hasMore ? ordered.Take(limit).ToImmutableArray() : ordered;
var nextCursor = hasMore ? EncodeCursor(page[^1]) : null;
var linkset = BuildAggregateLinkset(page);
RecordIdentifierCollisions(normalizedTenant, linkset);
return new AdvisoryObservationQueryResult(page, linkset, nextCursor, hasMore);
}
private static bool Matches(
AdvisoryObservation observation,
ImmutableHashSet<string> observationIds,
ImmutableHashSet<string> aliases,
ImmutableHashSet<string> purls,
ImmutableHashSet<string> cpes)
{
ArgumentNullException.ThrowIfNull(observation);
if (observationIds.Count > 0 && !observationIds.Contains(observation.ObservationId))
{
return false;
}
if (aliases.Count > 0 && !observation.Linkset.Aliases.Any(aliases.Contains))
{
return false;
}
if (purls.Count > 0 && !observation.Linkset.Purls.Any(purls.Contains))
{
return false;
}
if (cpes.Count > 0 && !observation.Linkset.Cpes.Any(cpes.Contains))
{
return false;
}
return true;
}
private static string NormalizeTenant(string tenant)
=> Validation.EnsureNotNullOrWhiteSpace(tenant, nameof(tenant)).ToLowerInvariant();
private static ImmutableHashSet<string> NormalizeSet(
IEnumerable<string>? values,
Func<string, string> projector,
StringComparer comparer)
{
if (values is null)
{
return ImmutableHashSet<string>.Empty;
}
var builder = ImmutableHashSet.CreateBuilder<string>(comparer);
foreach (var value in values)
{
var normalized = Validation.TrimToNull(value);
if (normalized is null)
{
continue;
}
builder.Add(projector(normalized));
}
return builder.ToImmutable();
}
private static int NormalizeLimit(int? requestedLimit)
{
if (!requestedLimit.HasValue || requestedLimit.Value <= 0)
{
return DefaultPageSize;
}
var limit = requestedLimit.Value;
if (limit > MaxPageSize)
{
return MaxPageSize;
}
return limit;
}
private static AdvisoryObservationCursor? DecodeCursor(string? cursor)
{
if (string.IsNullOrWhiteSpace(cursor))
{
return null;
}
try
{
var decoded = Convert.FromBase64String(cursor.Trim());
var payload = Encoding.UTF8.GetString(decoded);
var separator = payload.IndexOf(':');
if (separator <= 0 || separator >= payload.Length - 1)
{
throw new FormatException("Cursor is malformed.");
}
var ticksText = payload.AsSpan(0, separator);
if (!long.TryParse(ticksText, NumberStyles.Integer, CultureInfo.InvariantCulture, out var ticks))
{
throw new FormatException("Cursor timestamp is invalid.");
}
var createdAt = new DateTimeOffset(DateTime.SpecifyKind(new DateTime(ticks), DateTimeKind.Utc));
var observationId = payload[(separator + 1)..];
if (string.IsNullOrWhiteSpace(observationId))
{
throw new FormatException("Cursor observation id is missing.");
}
return new AdvisoryObservationCursor(createdAt, observationId);
}
catch (FormatException)
{
throw;
}
catch (Exception ex)
{
throw new FormatException("Cursor is malformed.", ex);
}
}
private static string? EncodeCursor(AdvisoryObservation observation)
{
if (observation is null)
{
return null;
}
var payload = $"{observation.CreatedAt.UtcTicks.ToString(CultureInfo.InvariantCulture)}:{observation.ObservationId}";
return Convert.ToBase64String(Encoding.UTF8.GetBytes(payload));
}
@@ -283,4 +285,18 @@ public sealed class AdvisoryObservationQueryService : IAdvisoryObservationQueryS
.ThenBy(static c => string.Join('|', c.Values ?? Array.Empty<string>()), StringComparer.Ordinal)
.ToImmutableArray());
}
private static void RecordIdentifierCollisions(string tenant, AdvisoryObservationLinksetAggregate linkset)
{
if (linkset.Conflicts.IsDefaultOrEmpty)
{
return;
}
var collisionCount = linkset.Conflicts.Count(conflict =>
string.Equals(conflict.Field, "aliases", StringComparison.OrdinalIgnoreCase) &&
conflict.Reason.Contains("alias", StringComparison.OrdinalIgnoreCase));
VulnExplorerTelemetry.RecordIdentifierCollisions(tenant, source: null, collisionCount);
}
}

2
src/Concelier/__Libraries/StellaOps.Concelier.Core/Risk/AdvisoryFieldChangeEmitter.cs
View File

@@ -5,6 +5,7 @@ using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Core.Diagnostics;
namespace StellaOps.Concelier.Core.Risk;
@@ -177,6 +178,7 @@ public sealed class AdvisoryFieldChangeEmitter : IAdvisoryFieldChangeEmitter
_logger.LogInformation(
"Emitted withdrawn observation notification for {ObservationId}",
previousSignal.ObservationId);
VulnExplorerTelemetry.RecordWithdrawnStatement(tenantId, previousSignal.Provenance.Vendor);
return notification;
}

1
src/Concelier/__Libraries/StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj
View File

@@ -12,6 +12,7 @@
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.0" />
<PackageReference Include="Cronos" Version="0.10.0" />
<PackageReference Include="System.Diagnostics.DiagnosticSource" Version="9.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj" />

16
src/Concelier/__Libraries/StellaOps.Concelier.Models/MongoCompat/DriverStubs.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
@@ -66,6 +67,8 @@ namespace MongoDB.Driver
public class MongoDatabase : IMongoDatabase
{
private readonly ConcurrentDictionary<string, object> _collections = new(StringComparer.Ordinal);
public MongoDatabase(string name)
{
Name = name;
@@ -73,8 +76,17 @@ namespace MongoDB.Driver
}
public string Name { get; }
public DatabaseNamespace DatabaseNamespace { get; }
public IMongoCollection<TDocument> GetCollection<TDocument>(string name, MongoCollectionSettings? settings = null) => new MongoCollection<TDocument>(name);
public Task DropCollectionAsync(string name, CancellationToken cancellationToken = default) => Task.CompletedTask;
public IMongoCollection<TDocument> GetCollection<TDocument>(string name, MongoCollectionSettings? settings = null)
{
var collection = (MongoCollection<TDocument>)_collections.GetOrAdd(name, _ => new MongoCollection<TDocument>(name));
return collection;
}
public Task DropCollectionAsync(string name, CancellationToken cancellationToken = default)
{
_collections.TryRemove(name, out _);
return Task.CompletedTask;
}
public BsonDocument RunCommand(BsonDocument command, CancellationToken cancellationToken = default) => new();
public T RunCommand<T>(BsonDocument command, CancellationToken cancellationToken = default) => default!;
public Task<T> RunCommandAsync<T>(BsonDocument command, CancellationToken cancellationToken = default) => Task.FromResult(default(T)!);

4
src/Concelier/__Libraries/StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj
View File

@@ -5,6 +5,7 @@
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>false</TreatWarningsAsErrors>
<EnableDefaultCompileItems>false</EnableDefaultCompileItems>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0" />
@@ -12,4 +13,7 @@
<ItemGroup>
<ProjectReference Include="..\StellaOps.Concelier.RawModels\StellaOps.Concelier.RawModels.csproj" />
</ItemGroup>
<ItemGroup>
<Compile Include="**\*.cs" Exclude="bin\**;obj\**;out\**;bin2\**" />
</ItemGroup>
</Project>

69
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Postgres/Migrations/005_connector_state.sql Normal file
View File

@@ -0,0 +1,69 @@
-- Concelier Migration 005: Postgres equivalents for DTO, export, PSIRT/JP flags, and change history.
CREATE SCHEMA IF NOT EXISTS concelier;
CREATE TABLE IF NOT EXISTS concelier.dtos (
id UUID NOT NULL,
document_id UUID NOT NULL,
source_name TEXT NOT NULL,
format TEXT NOT NULL,
payload_json JSONB NOT NULL,
schema_version TEXT NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
validated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
CONSTRAINT pk_concelier_dtos PRIMARY KEY (document_id)
);
CREATE INDEX IF NOT EXISTS idx_concelier_dtos_source ON concelier.dtos(source_name, created_at DESC);
CREATE TABLE IF NOT EXISTS concelier.export_states (
id TEXT NOT NULL,
export_cursor TEXT NOT NULL,
last_full_digest TEXT,
last_delta_digest TEXT,
base_export_id TEXT,
base_digest TEXT,
target_repository TEXT,
files JSONB NOT NULL,
exporter_version TEXT NOT NULL,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
CONSTRAINT pk_concelier_export_states PRIMARY KEY (id)
);
CREATE TABLE IF NOT EXISTS concelier.psirt_flags (
advisory_id TEXT NOT NULL,
vendor TEXT NOT NULL,
source_name TEXT NOT NULL,
external_id TEXT,
recorded_at TIMESTAMPTZ NOT NULL,
CONSTRAINT pk_concelier_psirt_flags PRIMARY KEY (advisory_id, vendor)
);
CREATE INDEX IF NOT EXISTS idx_concelier_psirt_source ON concelier.psirt_flags(source_name, recorded_at DESC);
CREATE TABLE IF NOT EXISTS concelier.jp_flags (
advisory_key TEXT NOT NULL,
source_name TEXT NOT NULL,
category TEXT NOT NULL,
vendor_status TEXT,
created_at TIMESTAMPTZ NOT NULL,
CONSTRAINT pk_concelier_jp_flags PRIMARY KEY (advisory_key)
);
CREATE TABLE IF NOT EXISTS concelier.change_history (
id UUID NOT NULL,
source_name TEXT NOT NULL,
advisory_key TEXT NOT NULL,
document_id UUID NOT NULL,
document_hash TEXT NOT NULL,
snapshot_hash TEXT NOT NULL,
previous_snapshot_hash TEXT,
snapshot JSONB NOT NULL,
previous_snapshot JSONB,
changes JSONB NOT NULL,
created_at TIMESTAMPTZ NOT NULL,
CONSTRAINT pk_concelier_change_history PRIMARY KEY (id)
);
CREATE INDEX IF NOT EXISTS idx_concelier_change_history_advisory
ON concelier.change_history(advisory_key, created_at DESC);

96
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Postgres/Repositories/PostgresChangeHistoryStore.cs Normal file
View File

@@ -0,0 +1,96 @@
using System.Text.Json;
using Dapper;
using StellaOps.Concelier.Storage.Mongo.ChangeHistory;
namespace StellaOps.Concelier.Storage.Postgres.Repositories;
internal sealed class PostgresChangeHistoryStore : IChangeHistoryStore
{
private readonly ConcelierDataSource _dataSource;
private readonly JsonSerializerOptions _jsonOptions = new(JsonSerializerDefaults.General)
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
};
public PostgresChangeHistoryStore(ConcelierDataSource dataSource)
{
_dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource));
}
public async Task AddAsync(ChangeHistoryRecord record, CancellationToken cancellationToken)
{
const string sql = """
INSERT INTO concelier.change_history
(id, source_name, advisory_key, document_id, document_hash, snapshot_hash, previous_snapshot_hash, snapshot, previous_snapshot, changes, created_at)
VALUES (@Id, @SourceName, @AdvisoryKey, @DocumentId, @DocumentHash, @SnapshotHash, @PreviousSnapshotHash, @Snapshot, @PreviousSnapshot, @Changes, @CreatedAt)
ON CONFLICT (id) DO NOTHING;
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
await connection.ExecuteAsync(new CommandDefinition(sql, new
{
record.Id,
record.SourceName,
record.AdvisoryKey,
record.DocumentId,
record.DocumentHash,
record.SnapshotHash,
record.PreviousSnapshotHash,
Snapshot = record.Snapshot,
PreviousSnapshot = record.PreviousSnapshot,
Changes = JsonSerializer.Serialize(record.Changes, _jsonOptions),
record.CreatedAt
}, cancellationToken: cancellationToken));
}
public async Task<IReadOnlyList<ChangeHistoryRecord>> GetRecentAsync(string sourceName, string advisoryKey, int limit, CancellationToken cancellationToken)
{
const string sql = """
SELECT id, source_name, advisory_key, document_id, document_hash, snapshot_hash, previous_snapshot_hash, snapshot, previous_snapshot, changes, created_at
FROM concelier.change_history
WHERE source_name = @SourceName AND advisory_key = @AdvisoryKey
ORDER BY created_at DESC
LIMIT @Limit;
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
var rows = await connection.QueryAsync<ChangeHistoryRow>(new CommandDefinition(sql, new
{
SourceName = sourceName,
AdvisoryKey = advisoryKey,
Limit = limit
}, cancellationToken: cancellationToken));
return rows.Select(ToRecord).ToArray();
}
private ChangeHistoryRecord ToRecord(ChangeHistoryRow row)
{
var changes = JsonSerializer.Deserialize<IReadOnlyList<ChangeHistoryFieldChange>>(row.Changes, _jsonOptions) ?? Array.Empty<ChangeHistoryFieldChange>();
return new ChangeHistoryRecord(
row.Id,
row.SourceName,
row.AdvisoryKey,
row.DocumentId,
row.DocumentHash,
row.SnapshotHash,
row.PreviousSnapshotHash ?? string.Empty,
row.Snapshot,
row.PreviousSnapshot ?? string.Empty,
changes,
row.CreatedAt);
}
private sealed record ChangeHistoryRow(
Guid Id,
string SourceName,
string AdvisoryKey,
Guid DocumentId,
string DocumentHash,
string SnapshotHash,
string? PreviousSnapshotHash,
string Snapshot,
string? PreviousSnapshot,
string Changes,
DateTimeOffset CreatedAt);
}

104
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Postgres/Repositories/PostgresDtoStore.cs Normal file
View File

@@ -0,0 +1,104 @@
using System.Text.Json;
using Dapper;
using StellaOps.Concelier.Storage.Mongo;
namespace StellaOps.Concelier.Storage.Postgres.Repositories;
internal sealed class PostgresDtoStore : IDtoStore
{
private readonly ConcelierDataSource _dataSource;
private readonly JsonSerializerOptions _jsonOptions = new(JsonSerializerDefaults.General)
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
};
public PostgresDtoStore(ConcelierDataSource dataSource)
{
_dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource));
}
public async Task<DtoRecord> UpsertAsync(DtoRecord record, CancellationToken cancellationToken)
{
const string sql = """
INSERT INTO concelier.dtos (id, document_id, source_name, format, payload_json, schema_version, created_at, validated_at)
VALUES (@Id, @DocumentId, @SourceName, @Format, @PayloadJson, @SchemaVersion, @CreatedAt, @ValidatedAt)
ON CONFLICT (document_id) DO UPDATE
SET payload_json = EXCLUDED.payload_json,
schema_version = EXCLUDED.schema_version,
source_name = EXCLUDED.source_name,
format = EXCLUDED.format,
validated_at = EXCLUDED.validated_at
RETURNING id, document_id, source_name, format, payload_json, schema_version, created_at, validated_at;
""";
var payloadJson = record.Payload.ToJson();
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
var row = await connection.QuerySingleAsync<DtoRow>(new CommandDefinition(sql, new
{
record.Id,
record.DocumentId,
record.SourceName,
record.Format,
PayloadJson = payloadJson,
record.SchemaVersion,
record.CreatedAt,
record.ValidatedAt
}, cancellationToken: cancellationToken));
return ToRecord(row);
}
public async Task<DtoRecord?> FindByDocumentIdAsync(Guid documentId, CancellationToken cancellationToken)
{
const string sql = """
SELECT id, document_id, source_name, format, payload_json, schema_version, created_at, validated_at
FROM concelier.dtos
WHERE document_id = @DocumentId
LIMIT 1;
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
var row = await connection.QuerySingleOrDefaultAsync<DtoRow>(new CommandDefinition(sql, new { DocumentId = documentId }, cancellationToken: cancellationToken));
return row is null ? null : ToRecord(row);
}
public async Task<IReadOnlyList<DtoRecord>> GetBySourceAsync(string sourceName, int limit, CancellationToken cancellationToken)
{
const string sql = """
SELECT id, document_id, source_name, format, payload_json, schema_version, created_at, validated_at
FROM concelier.dtos
WHERE source_name = @SourceName
ORDER BY created_at DESC
LIMIT @Limit;
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
var rows = await connection.QueryAsync<DtoRow>(new CommandDefinition(sql, new { SourceName = sourceName, Limit = limit }, cancellationToken: cancellationToken));
return rows.Select(ToRecord).ToArray();
}
private DtoRecord ToRecord(DtoRow row)
{
var payload = MongoDB.Bson.BsonDocument.Parse(row.PayloadJson);
return new DtoRecord(
row.Id,
row.DocumentId,
row.SourceName,
row.Format,
payload,
row.CreatedAt,
row.SchemaVersion,
row.ValidatedAt);
}
private sealed record DtoRow(
Guid Id,
Guid DocumentId,
string SourceName,
string Format,
string PayloadJson,
string SchemaVersion,
DateTimeOffset CreatedAt,
DateTimeOffset ValidatedAt);
}

119
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Postgres/Repositories/PostgresExportStateStore.cs Normal file
View File

@@ -0,0 +1,119 @@
using System.Text.Json;
using Dapper;
using StellaOps.Concelier.Storage.Mongo.Exporting;
namespace StellaOps.Concelier.Storage.Postgres.Repositories;
internal sealed class PostgresExportStateStore : IExportStateStore
{
private readonly ConcelierDataSource _dataSource;
private readonly JsonSerializerOptions _jsonOptions = new(JsonSerializerDefaults.General)
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
};
public PostgresExportStateStore(ConcelierDataSource dataSource)
{
_dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource));
}
public async Task<ExportStateRecord?> FindAsync(string id, CancellationToken cancellationToken)
{
const string sql = """
SELECT id,
export_cursor,
last_full_digest,
last_delta_digest,
base_export_id,
base_digest,
target_repository,
files,
exporter_version,
updated_at
FROM concelier.export_states
WHERE id = @Id
LIMIT 1;
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
var row = await connection.QuerySingleOrDefaultAsync<ExportStateRow>(new CommandDefinition(sql, new { Id = id }, cancellationToken: cancellationToken));
return row is null ? null : ToRecord(row);
}
public async Task<ExportStateRecord> UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken)
{
const string sql = """
INSERT INTO concelier.export_states
(id, export_cursor, last_full_digest, last_delta_digest, base_export_id, base_digest, target_repository, files, exporter_version, updated_at)
VALUES (@Id, @ExportCursor, @LastFullDigest, @LastDeltaDigest, @BaseExportId, @BaseDigest, @TargetRepository, @Files, @ExporterVersion, @UpdatedAt)
ON CONFLICT (id) DO UPDATE
SET export_cursor = EXCLUDED.export_cursor,
last_full_digest = EXCLUDED.last_full_digest,
last_delta_digest = EXCLUDED.last_delta_digest,
base_export_id = EXCLUDED.base_export_id,
base_digest = EXCLUDED.base_digest,
target_repository = EXCLUDED.target_repository,
files = EXCLUDED.files,
exporter_version = EXCLUDED.exporter_version,
updated_at = EXCLUDED.updated_at
RETURNING id,
export_cursor,
last_full_digest,
last_delta_digest,
base_export_id,
base_digest,
target_repository,
files,
exporter_version,
updated_at;
""";
var filesJson = JsonSerializer.Serialize(record.Files, _jsonOptions);
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
var row = await connection.QuerySingleAsync<ExportStateRow>(new CommandDefinition(sql, new
{
record.Id,
record.ExportCursor,
record.LastFullDigest,
record.LastDeltaDigest,
record.BaseExportId,
record.BaseDigest,
record.TargetRepository,
Files = filesJson,
record.ExporterVersion,
record.UpdatedAt
}, cancellationToken: cancellationToken));
return ToRecord(row);
}
private ExportStateRecord ToRecord(ExportStateRow row)
{
var files = JsonSerializer.Deserialize<IReadOnlyList<ExportFileRecord>>(row.Files, _jsonOptions) ?? Array.Empty<ExportFileRecord>();
return new ExportStateRecord(
row.Id,
row.ExportCursor,
row.LastFullDigest,
row.LastDeltaDigest,
row.BaseExportId,
row.BaseDigest,
row.TargetRepository,
files,
row.ExporterVersion,
row.UpdatedAt);
}
private sealed record ExportStateRow(
string Id,
string ExportCursor,
string? LastFullDigest,
string? LastDeltaDigest,
string? BaseExportId,
string? BaseDigest,
string? TargetRepository,
string Files,
string ExporterVersion,
DateTimeOffset UpdatedAt);
}

58
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Postgres/Repositories/PostgresJpFlagStore.cs Normal file
View File

@@ -0,0 +1,58 @@
using Dapper;
using StellaOps.Concelier.Storage.Mongo.JpFlags;
namespace StellaOps.Concelier.Storage.Postgres.Repositories;
internal sealed class PostgresJpFlagStore : IJpFlagStore
{
private readonly ConcelierDataSource _dataSource;
public PostgresJpFlagStore(ConcelierDataSource dataSource)
{
_dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource));
}
public async Task UpsertAsync(JpFlagRecord record, CancellationToken cancellationToken)
{
const string sql = """
INSERT INTO concelier.jp_flags (advisory_key, source_name, category, vendor_status, created_at)
VALUES (@AdvisoryKey, @SourceName, @Category, @VendorStatus, @CreatedAt)
ON CONFLICT (advisory_key) DO UPDATE
SET source_name = EXCLUDED.source_name,
category = EXCLUDED.category,
vendor_status = EXCLUDED.vendor_status,
created_at = EXCLUDED.created_at;
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
await connection.ExecuteAsync(new CommandDefinition(sql, new
{
record.AdvisoryKey,
record.SourceName,
record.Category,
record.VendorStatus,
record.CreatedAt
}, cancellationToken: cancellationToken));
}
public async Task<JpFlagRecord?> FindAsync(string advisoryKey, CancellationToken cancellationToken)
{
const string sql = """
SELECT advisory_key, source_name, category, vendor_status, created_at
FROM concelier.jp_flags
WHERE advisory_key = @AdvisoryKey
LIMIT 1;
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
var row = await connection.QuerySingleOrDefaultAsync<JpFlagRow>(new CommandDefinition(sql, new { AdvisoryKey = advisoryKey }, cancellationToken: cancellationToken));
return row is null ? null : new JpFlagRecord(row.AdvisoryKey, row.SourceName, row.Category, row.VendorStatus, row.CreatedAt);
}
private sealed record JpFlagRow(
string AdvisoryKey,
string SourceName,
string Category,
string? VendorStatus,
DateTimeOffset CreatedAt);
}

76
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Postgres/Repositories/PostgresPsirtFlagStore.cs Normal file
View File

@@ -0,0 +1,76 @@
using Dapper;
using StellaOps.Concelier.Storage.Mongo.PsirtFlags;
namespace StellaOps.Concelier.Storage.Postgres.Repositories;
internal sealed class PostgresPsirtFlagStore : IPsirtFlagStore
{
private readonly ConcelierDataSource _dataSource;
public PostgresPsirtFlagStore(ConcelierDataSource dataSource)
{
_dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource));
}
public async Task UpsertAsync(PsirtFlagRecord flag, CancellationToken cancellationToken)
{
const string sql = """
INSERT INTO concelier.psirt_flags (advisory_id, vendor, source_name, external_id, recorded_at)
VALUES (@AdvisoryId, @Vendor, @SourceName, @ExternalId, @RecordedAt)
ON CONFLICT (advisory_id, vendor) DO UPDATE
SET source_name = EXCLUDED.source_name,
external_id = EXCLUDED.external_id,
recorded_at = EXCLUDED.recorded_at;
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
await connection.ExecuteAsync(new CommandDefinition(sql, new
{
flag.AdvisoryId,
flag.Vendor,
flag.SourceName,
flag.ExternalId,
flag.RecordedAt
}, cancellationToken: cancellationToken));
}
public async Task<IReadOnlyList<PsirtFlagRecord>> GetRecentAsync(string advisoryKey, int limit, CancellationToken cancellationToken)
{
const string sql = """
SELECT advisory_id, vendor, source_name, external_id, recorded_at
FROM concelier.psirt_flags
WHERE advisory_id = @AdvisoryId
ORDER BY recorded_at DESC
LIMIT @Limit;
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
var rows = await connection.QueryAsync<PsirtFlagRow>(new CommandDefinition(sql, new { AdvisoryId = advisoryKey, Limit = limit }, cancellationToken: cancellationToken));
return rows.Select(ToRecord).ToArray();
}
public async Task<PsirtFlagRecord?> FindAsync(string advisoryKey, CancellationToken cancellationToken)
{
const string sql = """
SELECT advisory_id, vendor, source_name, external_id, recorded_at
FROM concelier.psirt_flags
WHERE advisory_id = @AdvisoryId
ORDER BY recorded_at DESC
LIMIT 1;
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
var row = await connection.QuerySingleOrDefaultAsync<PsirtFlagRow>(new CommandDefinition(sql, new { AdvisoryId = advisoryKey }, cancellationToken: cancellationToken));
return row is null ? null : ToRecord(row);
}
private static PsirtFlagRecord ToRecord(PsirtFlagRow row) =>
new(row.AdvisoryId, row.Vendor, row.SourceName, row.ExternalId, row.RecordedAt);
private sealed record PsirtFlagRow(
string AdvisoryId,
string Vendor,
string SourceName,
string? ExternalId,
DateTimeOffset RecordedAt);
}

14
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Postgres/ServiceCollectionExtensions.cs
View File

@@ -7,6 +7,10 @@ using StellaOps.Infrastructure.Postgres.Options;
using StellaOps.Concelier.Core.Linksets;
using MongoContracts = StellaOps.Concelier.Storage.Mongo;
using MongoAdvisories = StellaOps.Concelier.Storage.Mongo.Advisories;
using MongoExporting = StellaOps.Concelier.Storage.Mongo.Exporting;
using MongoJpFlags = StellaOps.Concelier.Storage.Mongo.JpFlags;
using MongoPsirt = StellaOps.Concelier.Storage.Mongo.PsirtFlags;
using MongoHistory = StellaOps.Concelier.Storage.Mongo.ChangeHistory;
namespace StellaOps.Concelier.Storage.Postgres;
@@ -51,6 +55,11 @@ public static class ServiceCollectionExtensions
services.AddScoped<IAdvisoryLinksetStore, AdvisoryLinksetCacheRepository>();
services.AddScoped<IAdvisoryLinksetLookup>(sp => sp.GetRequiredService<IAdvisoryLinksetStore>());
services.AddScoped<MongoContracts.IDocumentStore, PostgresDocumentStore>();
services.AddScoped<MongoContracts.IDtoStore, PostgresDtoStore>();
services.AddScoped<MongoExporting.IExportStateStore, PostgresExportStateStore>();
services.AddScoped<MongoPsirt.IPsirtFlagStore, PostgresPsirtFlagStore>();
services.AddScoped<MongoJpFlags.IJpFlagStore, PostgresJpFlagStore>();
services.AddScoped<MongoHistory.IChangeHistoryStore, PostgresChangeHistoryStore>();
return services;
}
@@ -89,6 +98,11 @@ public static class ServiceCollectionExtensions
services.AddScoped<IAdvisoryLinksetStore, AdvisoryLinksetCacheRepository>();
services.AddScoped<IAdvisoryLinksetLookup>(sp => sp.GetRequiredService<IAdvisoryLinksetStore>());
services.AddScoped<MongoContracts.IDocumentStore, PostgresDocumentStore>();
services.AddScoped<MongoContracts.IDtoStore, PostgresDtoStore>();
services.AddScoped<MongoExporting.IExportStateStore, PostgresExportStateStore>();
services.AddScoped<MongoPsirt.IPsirtFlagStore, PostgresPsirtFlagStore>();
services.AddScoped<MongoJpFlags.IJpFlagStore, PostgresJpFlagStore>();
services.AddScoped<MongoHistory.IChangeHistoryStore, PostgresChangeHistoryStore>();
return services;
}

1
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Postgres/StellaOps.Concelier.Storage.Postgres.csproj
View File

@@ -30,6 +30,7 @@
<ItemGroup>
<ProjectReference Include="..\..\..\__Libraries\StellaOps.Infrastructure.Postgres\StellaOps.Infrastructure.Postgres.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Core\StellaOps.Concelier.Core.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj" />
<ProjectReference Include="..\..\..\__Libraries\StellaOps.DependencyInjection\StellaOps.DependencyInjection.csproj" />
</ItemGroup>

107
src/Concelier/__Libraries/StellaOps.Concelier.Testing/MongoIntegrationFixture.cs
View File

@@ -1,81 +1,60 @@
using System;
using System.IO;
using System.Linq;
using MongoDB.Bson;
using System.Threading;
using System.Threading.Tasks;
using Mongo2Go;
using Xunit;
using MongoDB.Driver;
using Xunit;
namespace StellaOps.Concelier.Testing;
public sealed class MongoIntegrationFixture : IAsyncLifetime
{
public MongoDbRunner Runner { get; private set; } = null!;
public IMongoDatabase Database { get; private set; } = null!;
public IMongoClient Client { get; private set; } = null!;
public Task InitializeAsync()
/// <summary>
/// In-memory stand-in for the legacy Mongo2Go fixture. No external processes are launched;
/// DropDatabaseAsync simply resets the backing in-memory collections.
/// </summary>
public sealed class MongoIntegrationFixture : IAsyncLifetime
{
private readonly FixtureMongoClient _client;
private MongoDatabase _database;
public MongoIntegrationFixture()
{
EnsureMongo2GoEnvironment();
Runner = MongoDbRunner.Start(singleNodeReplSet: true);
Client = new MongoClient(Runner.ConnectionString);
Database = Client.GetDatabase($"concelier-tests-{Guid.NewGuid():N}");
return Task.CompletedTask;
}
public Task DisposeAsync()
{
Runner.Dispose();
return Task.CompletedTask;
_client = new FixtureMongoClient(this);
Runner = MongoDbRunner.Start(singleNodeReplSet: false);
_database = CreateDatabase();
}
private static void EnsureMongo2GoEnvironment()
public MongoDbRunner Runner { get; }
public IMongoDatabase Database => _database;
public IMongoClient Client => _client;
public Task InitializeAsync() => Task.CompletedTask;
public Task DisposeAsync() => Task.CompletedTask;
internal void Reset()
{
if (!OperatingSystem.IsLinux())
{
return;
}
var libraryPath = ResolveOpenSslLibraryPath();
if (libraryPath is null)
{
return;
}
var existing = Environment.GetEnvironmentVariable("LD_LIBRARY_PATH");
if (string.IsNullOrEmpty(existing))
{
Environment.SetEnvironmentVariable("LD_LIBRARY_PATH", libraryPath);
return;
}
var segments = existing.Split(':', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (!segments.Contains(libraryPath, StringComparer.Ordinal))
{
Environment.SetEnvironmentVariable("LD_LIBRARY_PATH", string.Join(':', new[] { libraryPath }.Concat(segments)));
}
_database = CreateDatabase();
}
private static string? ResolveOpenSslLibraryPath()
private MongoDatabase CreateDatabase() => new($"concelier-tests-{Guid.NewGuid():N}");
private sealed class FixtureMongoClient : IMongoClient
{
var current = AppContext.BaseDirectory;
while (!string.IsNullOrEmpty(current))
private readonly MongoIntegrationFixture _fixture;
public FixtureMongoClient(MongoIntegrationFixture fixture)
{
var candidate = Path.Combine(current, "tools", "openssl", "linux-x64");
if (Directory.Exists(candidate))
{
return candidate;
}
var parent = Directory.GetParent(current);
if (parent is null)
{
break;
}
current = parent.FullName;
_fixture = fixture;
}
return null;
public IMongoDatabase GetDatabase(string name, MongoDatabaseSettings? settings = null) => _fixture.Database;
public Task DropDatabaseAsync(string name, CancellationToken cancellationToken = default)
{
_fixture.Reset();
return Task.CompletedTask;
}
}
}

211
src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Diagnostics/VulnExplorerTelemetryTests.cs Normal file
View File

@@ -0,0 +1,211 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics.Metrics;
using System.Linq;
using System.Text.Json.Nodes;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Diagnostics;
using StellaOps.Concelier.Core.Observations;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.RawModels;
using Xunit;
namespace StellaOps.Concelier.Core.Tests.Diagnostics;
public sealed class VulnExplorerTelemetryTests
{
private static readonly AdvisoryObservationSource DefaultSource = new("ghsa", "stream", "https://example.test/api");
private static readonly AdvisoryObservationSignature DefaultSignature = new(false, null, null, null);
[Fact]
public async Task QueryAsync_RecordsIdentifierCollisionMetric()
{
var (listener, measurements) = CreateListener(
VulnExplorerTelemetry.MeterName,
"vuln.identifier_collisions_total");
var observations = new[]
{
CreateObservation(
"tenant-a:ghsa:1",
"tenant-a",
aliases: new[] { "CVE-2025-0001" }),
CreateObservation(
"tenant-a:osv:2",
"tenant-a",
aliases: new[] { "GHSA-aaaa-bbbb-cccc" })
};
var service = new AdvisoryObservationQueryService(new TestObservationLookup(observations));
await service.QueryAsync(new AdvisoryObservationQueryOptions("tenant-a"), CancellationToken.None);
listener.Dispose();
var collision = measurements.Single(m => m.Instrument == "vuln.identifier_collisions_total");
Assert.Equal(1, collision.Value);
Assert.Equal("tenant-a", collision.Tags.Single(t => t.Key == "tenant").Value);
}
[Fact]
public void RecordChunkRequest_EmitsCounterAndLatency()
{
var (listener, measurements) = CreateListener(
VulnExplorerTelemetry.MeterName,
"vuln.chunk_requests_total",
"vuln.chunk_latency_ms");
VulnExplorerTelemetry.RecordChunkRequest("tenant-a", "ok", cacheHit: true, chunkCount: 3, latencyMs: 42.5);
listener.Dispose();
Assert.Equal(1, measurements.Single(m => m.Instrument == "vuln.chunk_requests_total").Value);
Assert.Equal(42.5, measurements.Single(m => m.Instrument == "vuln.chunk_latency_ms").Value);
}
[Fact]
public void RecordWithdrawnStatement_EmitsCounter()
{
var (listener, measurements) = CreateListener(
VulnExplorerTelemetry.MeterName,
"vuln.withdrawn_statements_total");
VulnExplorerTelemetry.RecordWithdrawnStatement("tenant-a", "nvd");
listener.Dispose();
var withdrawn = measurements.Single(m => m.Instrument == "vuln.withdrawn_statements_total");
Assert.Equal(1, withdrawn.Value);
Assert.Equal("tenant-a", withdrawn.Tags.Single(t => t.Key == "tenant").Value);
Assert.Equal("nvd", withdrawn.Tags.Single(t => t.Key == "source").Value);
}
private static AdvisoryObservation CreateObservation(
string observationId,
string tenant,
IEnumerable<string>? aliases = null)
{
var upstream = new AdvisoryObservationUpstream(
upstreamId: $"upstream-{observationId}",
documentVersion: null,
fetchedAt: DateTimeOffset.UtcNow,
receivedAt: DateTimeOffset.UtcNow,
contentHash: "sha256:d41d8cd98f00b204e9800998ecf8427e",
signature: DefaultSignature);
var content = new AdvisoryObservationContent(
"json",
"1.0",
new JsonObject());
var aliasArray = aliases?.ToImmutableArray() ?? ImmutableArray<string>.Empty;
var linkset = new AdvisoryObservationLinkset(
aliasArray,
Enumerable.Empty<string>(),
Enumerable.Empty<string>(),
Enumerable.Empty<AdvisoryObservationReference>());
var rawLinkset = new RawLinkset
{
Aliases = aliasArray
};
return new AdvisoryObservation(
observationId,
tenant,
DefaultSource,
upstream,
content,
linkset,
rawLinkset,
DateTimeOffset.UtcNow);
}
private static (MeterListener Listener, List<MeasurementRecord> Measurements) CreateListener(
string meterName,
params string[] instruments)
{
var measurements = new List<MeasurementRecord>();
var instrumentSet = instruments.ToHashSet(StringComparer.Ordinal);
var listener = new MeterListener
{
InstrumentPublished = (instrument, meterListener) =>
{
if (string.Equals(instrument.Meter.Name, meterName, StringComparison.Ordinal) &&
instrumentSet.Contains(instrument.Name))
{
meterListener.EnableMeasurementEvents(instrument);
}
}
};
listener.SetMeasurementEventCallback<long>((instrument, measurement, tags, state) =>
{
if (instrumentSet.Contains(instrument.Name))
{
measurements.Add(new MeasurementRecord(instrument.Name, measurement, CopyTags(tags)));
}
});
listener.SetMeasurementEventCallback<double>((instrument, measurement, tags, state) =>
{
if (instrumentSet.Contains(instrument.Name))
{
measurements.Add(new MeasurementRecord(instrument.Name, measurement, CopyTags(tags)));
}
});
listener.Start();
return (listener, measurements);
}
private static IReadOnlyList<KeyValuePair<string, object?>> CopyTags(ReadOnlySpan<KeyValuePair<string, object?>> tags)
{
var list = new List<KeyValuePair<string, object?>>(tags.Length);
foreach (var tag in tags)
{
list.Add(tag);
}
return list;
}
private sealed record MeasurementRecord(string Instrument, double Value, IReadOnlyList<KeyValuePair<string, object?>> Tags);
private sealed class TestObservationLookup : IAdvisoryObservationLookup
{
private readonly IReadOnlyList<AdvisoryObservation> _observations;
public TestObservationLookup(IReadOnlyList<AdvisoryObservation> observations)
{
_observations = observations;
}
public ValueTask<IReadOnlyList<AdvisoryObservation>> ListByTenantAsync(string tenant, CancellationToken cancellationToken)
{
var matches = _observations
.Where(o => string.Equals(o.Tenant, tenant, StringComparison.OrdinalIgnoreCase))
.ToList();
return ValueTask.FromResult<IReadOnlyList<AdvisoryObservation>>(matches);
}
public ValueTask<IReadOnlyList<AdvisoryObservation>> FindByFiltersAsync(
string tenant,
IReadOnlyCollection<string> observationIds,
IReadOnlyCollection<string> aliases,
IReadOnlyCollection<string> purls,
IReadOnlyCollection<string> cpes,
AdvisoryObservationCursor? cursor,
int limit,
CancellationToken cancellationToken)
{
var matches = _observations
.Where(o => string.Equals(o.Tenant, tenant, StringComparison.OrdinalIgnoreCase))
.Take(limit)
.ToList();
return ValueTask.FromResult<IReadOnlyList<AdvisoryObservation>>(matches);
}
}
}

95
src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/VulnExplorerTelemetryTests.cs Normal file
View File

@@ -0,0 +1,95 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
using System.Text.Json;
using StellaOps.Concelier.Core.Diagnostics;
using StellaOps.Concelier.Core.Linksets;
using Xunit;
namespace StellaOps.Concelier.WebService.Tests;
public sealed class VulnExplorerTelemetryTests : IDisposable
{
private readonly MeterListener _listener;
private readonly List<(string Name, double Value, KeyValuePair<string, object?>[] Tags)> _histogramMeasurements = new();
private readonly List<(string Name, long Value, KeyValuePair<string, object?>[] Tags)> _counterMeasurements = new();
public VulnExplorerTelemetryTests()
{
_listener = new MeterListener
{
InstrumentPublished = (instrument, listener) =>
{
if (instrument.Meter.Name == VulnExplorerTelemetry.MeterName)
{
listener.EnableMeasurementEvents(instrument);
}
}
};
_listener.SetMeasurementEventCallback<long>((instrument, measurement, tags, state) =>
{
if (instrument.Meter.Name == VulnExplorerTelemetry.MeterName)
{
_counterMeasurements.Add((instrument.Name, measurement, tags.ToArray()));
}
});
_listener.SetMeasurementEventCallback<double>((instrument, measurement, tags, state) =>
{
if (instrument.Meter.Name == VulnExplorerTelemetry.MeterName)
{
_histogramMeasurements.Add((instrument.Name, measurement, tags.ToArray()));
}
});
_listener.Start();
}
[Fact]
public void CountAliasCollisions_FiltersAliasConflicts()
{
var conflicts = new List<AdvisoryLinksetConflict>
{
new("aliases", "alias-inconsistency", Array.Empty<string>()),
new("ranges", "range-divergence", Array.Empty<string>()),
new("alias-field", "ALIAS-INCONSISTENCY", Array.Empty<string>())
};
var count = VulnExplorerTelemetry.CountAliasCollisions(conflicts);
Assert.Equal(2, count);
}
[Fact]
public void IsWithdrawn_DetectsWithdrawnFlagsAndTimestamps()
{
using var json = JsonDocument.Parse("{\"withdrawn\":true,\"withdrawn_at\":\"2024-10-10T00:00:00Z\"}");
Assert.True(VulnExplorerTelemetry.IsWithdrawn(json.RootElement));
}
[Fact]
public void RecordChunkLatency_EmitsHistogramMeasurement()
{
VulnExplorerTelemetry.RecordChunkLatency("tenant-a", "vendor-a", TimeSpan.FromMilliseconds(42));
var measurement = Assert.Single(_histogramMeasurements);
Assert.Equal("vuln.chunk_latency_ms", measurement.Name);
Assert.Equal(42, measurement.Value);
}
[Fact]
public void RecordWithdrawnStatement_EmitsCounter()
{
VulnExplorerTelemetry.RecordWithdrawnStatement("tenant-b", "vendor-b");
var measurement = Assert.Single(_counterMeasurements);
Assert.Equal("vuln.withdrawn_statements_total", measurement.Name);
Assert.Equal(1, measurement.Value);
}
public void Dispose()
{
_listener.Dispose();
}
}

82
src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/WebServiceEndpointsTests.cs
View File

@@ -75,16 +75,7 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
public Task InitializeAsync()
{
PrepareMongoEnvironment();
if (TryStartExternalMongo(out var externalConnectionString) && !string.IsNullOrWhiteSpace(externalConnectionString))
{
_factory = new ConcelierApplicationFactory(externalConnectionString);
}
else
{
_runner = MongoDbRunner.Start(singleNodeReplSet: true);
_factory = new ConcelierApplicationFactory(_runner.ConnectionString);
}
_factory = new ConcelierApplicationFactory(string.Empty);
WarmupFactory(_factory);
return Task.CompletedTask;
}
@@ -92,30 +83,6 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
public Task DisposeAsync()
{
_factory.Dispose();
if (_externalMongo is not null)
{
try
{
if (!_externalMongo.HasExited)
{
_externalMongo.Kill(true);
_externalMongo.WaitForExit(2000);
}
}
catch
{
// ignore cleanup errors in tests
}
if (!string.IsNullOrEmpty(_externalMongoDataPath) && Directory.Exists(_externalMongoDataPath))
{
try { Directory.Delete(_externalMongoDataPath, recursive: true); } catch { /* ignore */ }
}
}
else
{
_runner.Dispose();
}
return Task.CompletedTask;
}
@@ -141,12 +108,12 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
var healthPayload = await healthResponse.Content.ReadFromJsonAsync<HealthPayload>();
Assert.NotNull(healthPayload);
Assert.Equal("healthy", healthPayload!.Status);
Assert.Equal("mongo", healthPayload.Storage.Driver);
Assert.Equal("postgres", healthPayload.Storage.Backend);
var readyPayload = await readyResponse.Content.ReadFromJsonAsync<ReadyPayload>();
Assert.NotNull(readyPayload);
Assert.Equal("ready", readyPayload!.Status);
Assert.Equal("ready", readyPayload.Mongo.Status);
Assert.True(readyPayload!.Status is "ready" or "degraded");
Assert.Equal("postgres", readyPayload.Storage.Backend);
}
[Fact]
@@ -2019,9 +1986,10 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
private sealed class ConcelierApplicationFactory : WebApplicationFactory<Program>
{
private readonly string _connectionString;
private readonly string? _previousDsn;
private readonly string? _previousDriver;
private readonly string? _previousTimeout;
private readonly string? _previousPgDsn;
private readonly string? _previousPgEnabled;
private readonly string? _previousPgTimeout;
private readonly string? _previousPgSchema;
private readonly string? _previousTelemetryEnabled;
private readonly string? _previousTelemetryLogging;
private readonly string? _previousTelemetryTracing;
@@ -2035,11 +2003,15 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
Action<ConcelierOptions.AuthorityOptions>? authorityConfigure = null,
IDictionary<string, string?>? environmentOverrides = null)
{
_connectionString = connectionString;
var defaultPostgresDsn = "Host=localhost;Port=5432;Database=concelier_test;Username=postgres;Password=postgres";
_connectionString = string.IsNullOrWhiteSpace(connectionString) || connectionString.StartsWith("mongodb://", StringComparison.OrdinalIgnoreCase)
? defaultPostgresDsn
: connectionString;
_authorityConfigure = authorityConfigure;
_previousDsn = Environment.GetEnvironmentVariable("CONCELIER_STORAGE__DSN");
_previousDriver = Environment.GetEnvironmentVariable("CONCELIER_STORAGE__DRIVER");
_previousTimeout = Environment.GetEnvironmentVariable("CONCELIER_STORAGE__COMMANDTIMEOUTSECONDS");
_previousPgDsn = Environment.GetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__CONNECTIONSTRING");
_previousPgEnabled = Environment.GetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__ENABLED");
_previousPgTimeout = Environment.GetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__COMMANDTIMEOUTSECONDS");
_previousPgSchema = Environment.GetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__SCHEMANAME");
_previousTelemetryEnabled = Environment.GetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLED");
_previousTelemetryLogging = Environment.GetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLELOGGING");
_previousTelemetryTracing = Environment.GetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLETRACING");
@@ -2055,13 +2027,15 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
Environment.SetEnvironmentVariable("LD_LIBRARY_PATH", merged);
}
Environment.SetEnvironmentVariable("CONCELIER_STORAGE__DSN", connectionString);
Environment.SetEnvironmentVariable("CONCELIER_STORAGE__DRIVER", "mongo");
Environment.SetEnvironmentVariable("CONCELIER_STORAGE__COMMANDTIMEOUTSECONDS", "30");
Environment.SetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__CONNECTIONSTRING", _connectionString);
Environment.SetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__ENABLED", "true");
Environment.SetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__COMMANDTIMEOUTSECONDS", "30");
Environment.SetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__SCHEMANAME", "vuln");
Environment.SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLED", "false");
Environment.SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLELOGGING", "false");
Environment.SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLETRACING", "false");
Environment.SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLEMETRICS", "false");
Environment.SetEnvironmentVariable("CONCELIER_SKIP_OPTIONS_VALIDATION", "1");
const string EvidenceRootKey = "CONCELIER_EVIDENCE__ROOT";
var repoRoot = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..", "..", ".."));
_additionalPreviousEnvironment[EvidenceRootKey] = Environment.GetEnvironmentVariable(EvidenceRootKey);
@@ -2176,9 +2150,11 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
protected override void Dispose(bool disposing)
{
base.Dispose(disposing);
Environment.SetEnvironmentVariable("CONCELIER_STORAGE__DSN", _previousDsn);
Environment.SetEnvironmentVariable("CONCELIER_STORAGE__DRIVER", _previousDriver);
Environment.SetEnvironmentVariable("CONCELIER_STORAGE__COMMANDTIMEOUTSECONDS", _previousTimeout);
Environment.SetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__CONNECTIONSTRING", _previousPgDsn);
Environment.SetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__ENABLED", _previousPgEnabled);
Environment.SetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__COMMANDTIMEOUTSECONDS", _previousPgTimeout);
Environment.SetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__SCHEMANAME", _previousPgSchema);
Environment.SetEnvironmentVariable("CONCELIER_SKIP_OPTIONS_VALIDATION", null);
Environment.SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLED", _previousTelemetryEnabled);
Environment.SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLELOGGING", _previousTelemetryLogging);
Environment.SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLETRACING", _previousTelemetryTracing);
@@ -2470,13 +2446,11 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
private sealed record HealthPayload(string Status, DateTimeOffset StartedAt, double UptimeSeconds, StoragePayload Storage, TelemetryPayload Telemetry);
private sealed record StoragePayload(string Driver, bool Completed, DateTimeOffset? CompletedAt, double? DurationMs);
private sealed record StoragePayload(string Backend, bool Ready, DateTimeOffset? CheckedAt, double? LatencyMs, string? Error);
private sealed record TelemetryPayload(bool Enabled, bool Tracing, bool Metrics, bool Logging);
private sealed record ReadyPayload(string Status, DateTimeOffset StartedAt, double UptimeSeconds, ReadyMongoPayload Mongo);
private sealed record ReadyMongoPayload(string Status, double? LatencyMs, DateTimeOffset? CheckedAt, string? Error);
private sealed record ReadyPayload(string Status, DateTimeOffset StartedAt, double UptimeSeconds, StoragePayload Storage);
private sealed record JobDefinitionPayload(string Kind, bool Enabled, string? CronExpression, TimeSpan Timeout, TimeSpan LeaseDuration, JobRunPayload? LastRun);