Add SBOM, symbols, traces, and VEX files for CVE-2022-21661 SQLi case
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Created CycloneDX and SPDX SBOM files for both reachable and unreachable images.
- Added symbols.json detailing function entry and sink points in the WordPress code.
- Included runtime traces for function calls in both reachable and unreachable scenarios.
- Developed OpenVEX files indicating vulnerability status and justification for both cases.
- Updated README for evaluator harness to guide integration with scanner output.
This commit is contained in:
master
2025-11-08 20:53:45 +02:00
parent 515975edc5
commit 536f6249a6
837 changed files with 37279 additions and 14675 deletions

View File

@@ -6,6 +6,7 @@ ASP.NET Minimal API surface for Excititor ingest, provider administration, recon
- HTTP endpoints `/excititor/*` with authentication, authorization scopes, request validation, and deterministic responses.
- Job orchestration bridges for Worker hand-off (when co-hosted) and offline-friendly configuration.
- Observability (structured logs, metrics, tracing) aligned with StellaOps conventions.
- Optional/minor DI dependencies on minimal APIs must be declared with `[FromServices] SomeType? service = null` parameters so endpoint tests do not require bespoke service registrations.
## Participants
- StellaOps.Cli sends `excititor` verbs to this service via token-authenticated HTTPS.
- Worker receives scheduled jobs and uses shared infrastructure via common DI extensions.

View File

@@ -0,0 +1,114 @@
using System;
using System.Collections.Generic;
using System.Text.Json;
using System.Text.Json.Serialization;
using StellaOps.Concelier.RawModels;
using RawVexDocumentModel = StellaOps.Concelier.RawModels.VexRawDocument;
namespace StellaOps.Excititor.WebService.Contracts;
public sealed record VexIngestRequest(
[property: JsonPropertyName("providerId")] string ProviderId,
[property: JsonPropertyName("source")] VexIngestSourceRequest Source,
[property: JsonPropertyName("upstream")] VexIngestUpstreamRequest Upstream,
[property: JsonPropertyName("content")] VexIngestContentRequest Content,
[property: JsonPropertyName("metadata")] IReadOnlyDictionary<string, string>? Metadata);
public sealed record VexIngestSourceRequest(
[property: JsonPropertyName("vendor")] string Vendor,
[property: JsonPropertyName("connector")] string Connector,
[property: JsonPropertyName("version")] string Version,
[property: JsonPropertyName("stream")] string? Stream);
public sealed record VexIngestUpstreamRequest(
[property: JsonPropertyName("sourceUri")] string SourceUri,
[property: JsonPropertyName("upstreamId")] string UpstreamId,
[property: JsonPropertyName("documentVersion")] string? DocumentVersion,
[property: JsonPropertyName("retrievedAt")] DateTimeOffset? RetrievedAt,
[property: JsonPropertyName("contentHash")] string? ContentHash,
[property: JsonPropertyName("signature")] VexIngestSignatureRequest? Signature,
[property: JsonPropertyName("provenance")] IReadOnlyDictionary<string, string>? Provenance);
public sealed record VexIngestSignatureRequest(
[property: JsonPropertyName("present")] bool Present,
[property: JsonPropertyName("format")] string? Format,
[property: JsonPropertyName("keyId")] string? KeyId,
[property: JsonPropertyName("sig")] string? Signature,
[property: JsonPropertyName("certificate")] string? Certificate,
[property: JsonPropertyName("digest")] string? Digest);
public sealed record VexIngestContentRequest(
[property: JsonPropertyName("format")] string Format,
[property: JsonPropertyName("specVersion")] string? SpecVersion,
[property: JsonPropertyName("raw")] JsonElement Raw,
[property: JsonPropertyName("encoding")] string? Encoding);
public sealed record VexIngestResponse(
[property: JsonPropertyName("digest")] string Digest,
[property: JsonPropertyName("inserted")] bool Inserted,
[property: JsonPropertyName("tenant")] string Tenant,
[property: JsonPropertyName("retrievedAt")] DateTimeOffset RetrievedAt);
public sealed record VexRawSummaryResponse(
[property: JsonPropertyName("digest")] string Digest,
[property: JsonPropertyName("providerId")] string ProviderId,
[property: JsonPropertyName("format")] string Format,
[property: JsonPropertyName("sourceUri")] string SourceUri,
[property: JsonPropertyName("retrievedAt")] DateTimeOffset RetrievedAt,
[property: JsonPropertyName("inlineContent")] bool InlineContent,
[property: JsonPropertyName("metadata")] IReadOnlyDictionary<string, string> Metadata);
public sealed record VexRawListResponse(
[property: JsonPropertyName("records")] IReadOnlyList<VexRawSummaryResponse> Records,
[property: JsonPropertyName("nextCursor")] string? NextCursor,
[property: JsonPropertyName("hasMore")] bool HasMore);
public sealed record VexRawRecordResponse(
[property: JsonPropertyName("digest")] string Digest,
[property: JsonPropertyName("document")] RawVexDocumentModel Document,
[property: JsonPropertyName("retrievedAt")] DateTimeOffset RetrievedAt);
public sealed record VexRawProvenanceResponse(
[property: JsonPropertyName("digest")] string Digest,
[property: JsonPropertyName("tenant")] string Tenant,
[property: JsonPropertyName("source")] RawSourceMetadata Source,
[property: JsonPropertyName("upstream")] RawUpstreamMetadata Upstream,
[property: JsonPropertyName("retrievedAt")] DateTimeOffset RetrievedAt);
public sealed record VexAocVerifyRequest(
[property: JsonPropertyName("since")] DateTimeOffset? Since,
[property: JsonPropertyName("until")] DateTimeOffset? Until,
[property: JsonPropertyName("limit")] int? Limit,
[property: JsonPropertyName("sources")] IReadOnlyList<string>? Sources,
[property: JsonPropertyName("codes")] IReadOnlyList<string>? Codes);
public sealed record VexAocVerifyResponse(
[property: JsonPropertyName("tenant")] string Tenant,
[property: JsonPropertyName("window")] VexAocVerifyWindow Window,
[property: JsonPropertyName("checked")] VexAocVerifyChecked Checked,
[property: JsonPropertyName("violations")] IReadOnlyList<VexAocVerifyViolation> Violations,
[property: JsonPropertyName("metrics")] VexAocVerifyMetrics Metrics,
[property: JsonPropertyName("truncated")] bool Truncated);
public sealed record VexAocVerifyWindow(
[property: JsonPropertyName("from")] DateTimeOffset From,
[property: JsonPropertyName("to")] DateTimeOffset To);
public sealed record VexAocVerifyChecked(
[property: JsonPropertyName("advisories")] int Advisories,
[property: JsonPropertyName("vex")] int Vex);
public sealed record VexAocVerifyMetrics(
[property: JsonPropertyName("ingestion_write_total")] int IngestionWriteTotal,
[property: JsonPropertyName("aoc_violation_total")] int AocViolationTotal);
public sealed record VexAocVerifyViolation(
[property: JsonPropertyName("code")] string Code,
[property: JsonPropertyName("count")] int Count,
[property: JsonPropertyName("examples")] IReadOnlyList<VexAocVerifyViolationExample> Examples);
public sealed record VexAocVerifyViolationExample(
[property: JsonPropertyName("source")] string Source,
[property: JsonPropertyName("documentId")] string DocumentId,
[property: JsonPropertyName("contentHash")] string ContentHash,
[property: JsonPropertyName("path")] string Path);

View File

@@ -0,0 +1,76 @@
using System.Diagnostics;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Primitives;
using System.Threading.Tasks;
namespace StellaOps.Excititor.WebService.Extensions;
internal static class ObservabilityExtensions
{
private const string TraceHeaderName = "X-Stella-TraceId";
private const string CorrelationHeaderName = "X-Stella-CorrelationId";
private const string LegacyCorrelationHeaderName = "X-Correlation-Id";
private const string CorrelationItemKey = "__stella.correlationId";
public static IApplicationBuilder UseObservabilityHeaders(this IApplicationBuilder app)
{
return app.Use((context, next) =>
{
var correlationId = ResolveCorrelationId(context);
context.Items[CorrelationItemKey] = correlationId;
context.Response.OnStarting(state =>
{
var httpContext = (HttpContext)state;
ApplyHeaders(httpContext);
return Task.CompletedTask;
}, context);
return next();
});
}
private static void ApplyHeaders(HttpContext context)
{
var traceId = Activity.Current?.TraceId.ToString() ?? context.TraceIdentifier;
if (!string.IsNullOrWhiteSpace(traceId))
{
context.Response.Headers[TraceHeaderName] = traceId;
}
var correlationId = ResolveCorrelationId(context);
if (!string.IsNullOrWhiteSpace(correlationId))
{
context.Response.Headers[CorrelationHeaderName] = correlationId!;
}
}
private static string ResolveCorrelationId(HttpContext context)
{
if (context.Items.TryGetValue(CorrelationItemKey, out var existing) && existing is string cached && !string.IsNullOrWhiteSpace(cached))
{
return cached;
}
if (TryReadHeader(context.Request.Headers, CorrelationHeaderName, out var headerValue) ||
TryReadHeader(context.Request.Headers, LegacyCorrelationHeaderName, out headerValue))
{
return headerValue!;
}
return context.TraceIdentifier;
}
private static bool TryReadHeader(IHeaderDictionary headers, string name, out string? value)
{
if (headers.TryGetValue(name, out StringValues header) && !StringValues.IsNullOrEmpty(header))
{
value = header.ToString();
return true;
}
value = null;
return false;
}
}

View File

@@ -0,0 +1,140 @@
using System;
using System.Collections.Generic;
using System.Reflection;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.DependencyInjection;
using OpenTelemetry.Metrics;
using OpenTelemetry.Resources;
using OpenTelemetry.Trace;
using StellaOps.Excititor.WebService.Options;
using StellaOps.Ingestion.Telemetry;
namespace StellaOps.Excititor.WebService.Extensions;
internal static class TelemetryExtensions
{
public static void ConfigureExcititorTelemetry(this WebApplicationBuilder builder)
{
var telemetryOptions = new ExcititorTelemetryOptions();
builder.Configuration.GetSection("Excititor:Telemetry").Bind(telemetryOptions);
if (!telemetryOptions.Enabled || (!telemetryOptions.EnableTracing && !telemetryOptions.EnableMetrics))
{
return;
}
var openTelemetry = builder.Services.AddOpenTelemetry();
openTelemetry.ConfigureResource(resource =>
{
var serviceName = telemetryOptions.ServiceName ?? builder.Environment.ApplicationName ?? "StellaOps.Excititor.WebService";
var version = Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown";
resource.AddService(serviceName, serviceVersion: version, serviceInstanceId: Environment.MachineName);
foreach (var attribute in telemetryOptions.ResourceAttributes)
{
if (string.IsNullOrWhiteSpace(attribute.Key) || string.IsNullOrWhiteSpace(attribute.Value))
{
continue;
}
resource.AddAttributes(new[]
{
new KeyValuePair<string, object>(attribute.Key, attribute.Value)
});
}
});
if (telemetryOptions.EnableTracing)
{
openTelemetry.WithTracing(tracing =>
{
tracing
.AddSource(IngestionTelemetry.ActivitySourceName)
.AddAspNetCoreInstrumentation()
.AddHttpClientInstrumentation();
ConfigureExporters(telemetryOptions, tracing);
});
}
if (telemetryOptions.EnableMetrics)
{
openTelemetry.WithMetrics(metrics =>
{
metrics
.AddMeter(IngestionTelemetry.MeterName)
.AddAspNetCoreInstrumentation()
.AddHttpClientInstrumentation()
.AddRuntimeInstrumentation();
ConfigureExporters(telemetryOptions, metrics);
});
}
}
private static void ConfigureExporters(ExcititorTelemetryOptions options, TracerProviderBuilder tracing)
{
if (!string.IsNullOrWhiteSpace(options.OtlpEndpoint))
{
tracing.AddOtlpExporter(otlp =>
{
otlp.Endpoint = new Uri(options.OtlpEndpoint, UriKind.Absolute);
var headers = BuildHeaders(options.OtlpHeaders);
if (!string.IsNullOrEmpty(headers))
{
otlp.Headers = headers;
}
});
return;
}
if (options.ExportConsole)
{
tracing.AddConsoleExporter();
}
}
private static void ConfigureExporters(ExcititorTelemetryOptions options, MeterProviderBuilder metrics)
{
if (!string.IsNullOrWhiteSpace(options.OtlpEndpoint))
{
metrics.AddOtlpExporter(otlp =>
{
otlp.Endpoint = new Uri(options.OtlpEndpoint, UriKind.Absolute);
var headers = BuildHeaders(options.OtlpHeaders);
if (!string.IsNullOrEmpty(headers))
{
otlp.Headers = headers;
}
});
return;
}
if (options.ExportConsole)
{
metrics.AddConsoleExporter();
}
}
private static string? BuildHeaders(IReadOnlyDictionary<string, string> headers)
{
if (headers.Count == 0)
{
return null;
}
var parts = new List<string>(headers.Count);
foreach (var header in headers)
{
if (string.IsNullOrWhiteSpace(header.Key) || string.IsNullOrWhiteSpace(header.Value))
{
continue;
}
parts.Add($"{header.Key}={header.Value}");
}
return parts.Count == 0 ? null : string.Join(',', parts);
}
}

View File

@@ -0,0 +1,150 @@
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.WebService.Contracts;
namespace StellaOps.Excititor.WebService.Extensions;
internal static class VexRawRequestMapper
{
public static VexRawDocument Map(VexIngestRequest request, string tenant, TimeProvider timeProvider)
{
ArgumentNullException.ThrowIfNull(request);
ArgumentException.ThrowIfNullOrWhiteSpace(request.ProviderId);
ArgumentNullException.ThrowIfNull(timeProvider);
var source = request.Source ?? throw new ArgumentException("source section is required.", nameof(request));
var upstream = request.Upstream ?? throw new ArgumentException("upstream section is required.", nameof(request));
var content = request.Content ?? throw new ArgumentException("content section is required.", nameof(request));
ArgumentException.ThrowIfNullOrWhiteSpace(upstream.SourceUri);
ArgumentException.ThrowIfNullOrWhiteSpace(upstream.UpstreamId);
var providerId = request.ProviderId.Trim();
var format = ParseFormat(content.Format);
var sourceUri = new Uri(upstream.SourceUri!, UriKind.Absolute);
var retrievedAt = upstream.RetrievedAt ?? timeProvider.GetUtcNow();
var payload = SerializeContent(content.Raw);
var digest = NormalizeDigest(upstream.ContentHash, payload.Span);
var metadataBuilder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
CopyMetadata(metadataBuilder, request.Metadata);
CopyMetadata(metadataBuilder, upstream.Provenance);
metadataBuilder["tenant"] = tenant.Trim().ToLowerInvariant();
SetIfMissing(metadataBuilder, "source.vendor", source.Vendor);
SetIfMissing(metadataBuilder, "source.connector", source.Connector);
SetIfMissing(metadataBuilder, "source.connector_version", source.Version);
SetIfMissing(metadataBuilder, "source.stream", source.Stream ?? format.ToString().ToLowerInvariant());
SetIfMissing(metadataBuilder, "upstream.id", upstream.UpstreamId);
SetIfMissing(metadataBuilder, "upstream.version", upstream.DocumentVersion ?? retrievedAt.ToString("O"));
SetIfMissing(metadataBuilder, "content.spec_version", content.SpecVersion);
SetIfMissing(metadataBuilder, "content.encoding", content.Encoding);
var signature = upstream.Signature;
metadataBuilder["signature.present"] = (signature?.Present ?? false).ToString();
SetIfMissing(metadataBuilder, "signature.format", signature?.Format);
SetIfMissing(metadataBuilder, "signature.key_id", signature?.KeyId);
SetIfMissing(metadataBuilder, "signature.sig", signature?.Signature);
SetIfMissing(metadataBuilder, "signature.certificate", signature?.Certificate);
SetIfMissing(metadataBuilder, "signature.digest", signature?.Digest);
var metadata = metadataBuilder.ToImmutable();
return new VexRawDocument(
providerId,
format,
sourceUri,
retrievedAt,
digest,
payload,
metadata);
}
private static ReadOnlyMemory<byte> SerializeContent(JsonElement element)
{
var buffer = new ArrayBufferWriter<byte>();
using (var writer = new Utf8JsonWriter(buffer, new JsonWriterOptions { Indented = false }))
{
if (element.ValueKind == JsonValueKind.Undefined)
{
writer.WriteStartObject();
writer.WriteEndObject();
}
else
{
element.WriteTo(writer);
}
}
return buffer.WrittenMemory.ToArray();
}
private static VexDocumentFormat ParseFormat(string format)
{
if (string.IsNullOrWhiteSpace(format))
{
throw new ArgumentException("content.format is required.", nameof(format));
}
if (Enum.TryParse<VexDocumentFormat>(format, ignoreCase: true, out var parsed))
{
return parsed;
}
throw new ArgumentException($"Unsupported VEX document format {format}.", nameof(format));
}
private static string NormalizeDigest(string? existingDigest, ReadOnlySpan<byte> payload)
{
if (!string.IsNullOrWhiteSpace(existingDigest))
{
return existingDigest.Trim();
}
Span<byte> buffer = stackalloc byte[32];
if (SHA256.TryHashData(payload, buffer, out _))
{
return "sha256:" + Convert.ToHexString(buffer).ToLowerInvariant();
}
using var sha = SHA256.Create();
var hash = sha.ComputeHash(payload.ToArray());
return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant();
}
private static void CopyMetadata(ImmutableDictionary<string, string>.Builder builder, IReadOnlyDictionary<string, string>? values)
{
if (values is null)
{
return;
}
foreach (var kvp in values)
{
if (string.IsNullOrWhiteSpace(kvp.Key))
{
continue;
}
builder[kvp.Key.Trim()] = kvp.Value?.Trim() ?? string.Empty;
}
}
private static void SetIfMissing(ImmutableDictionary<string, string>.Builder builder, string key, string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return;
}
if (!builder.ContainsKey(key))
{
builder[key] = value.Trim();
}
}
}

View File

@@ -0,0 +1,53 @@
using System;
namespace StellaOps.Excititor.WebService.Options;
internal sealed class ExcititorObservabilityOptions
{
public TimeSpan IngestWarningThreshold { get; set; } = TimeSpan.FromHours(6);
public TimeSpan IngestCriticalThreshold { get; set; } = TimeSpan.FromHours(24);
public TimeSpan LinkWarningThreshold { get; set; } = TimeSpan.FromMinutes(15);
public TimeSpan LinkCriticalThreshold { get; set; } = TimeSpan.FromHours(1);
public TimeSpan SignatureWindow { get; set; } = TimeSpan.FromHours(12);
public double SignatureHealthyCoverage { get; set; } = 0.8;
public double SignatureWarningCoverage { get; set; } = 0.5;
public TimeSpan ConflictTrendWindow { get; set; } = TimeSpan.FromHours(24);
public int ConflictTrendBucketMinutes { get; set; } = 60;
public double ConflictWarningRatio { get; set; } = 0.15;
public double ConflictCriticalRatio { get; set; } = 0.3;
public int MaxConnectorDetails { get; set; } = 50;
internal TimeSpan GetPositive(TimeSpan candidate, TimeSpan fallback)
=> candidate <= TimeSpan.Zero ? fallback : candidate;
internal double ClampRatio(double value, double fallback)
{
if (double.IsNaN(value) || double.IsInfinity(value))
{
return fallback;
}
if (value < 0)
{
return 0;
}
if (value > 1)
{
return 1;
}
return value;
}
}

View File

@@ -0,0 +1,23 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Excititor.WebService.Options;
internal sealed class ExcititorTelemetryOptions
{
public bool Enabled { get; set; } = true;
public bool EnableTracing { get; set; } = true;
public bool EnableMetrics { get; set; } = true;
public bool ExportConsole { get; set; }
public string? ServiceName { get; set; }
public string? OtlpEndpoint { get; set; }
public Dictionary<string, string> OtlpHeaders { get; } = new(StringComparer.OrdinalIgnoreCase);
public Dictionary<string, string> ResourceAttributes { get; } = new(StringComparer.OrdinalIgnoreCase);
}

View File

@@ -0,0 +1,130 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using Microsoft.AspNetCore.Http;
using MongoDB.Bson;
using StellaOps.Excititor.Core.Aoc;
using StellaOps.Excititor.Storage.Mongo;
public partial class Program
{
private const string TenantHeaderName = "X-Stella-Tenant";
private static bool TryResolveTenant(HttpContext context, VexMongoStorageOptions options, bool requireHeader, out string tenant, out IResult? problem)
{
tenant = options.DefaultTenant;
problem = null;
if (context.Request.Headers.TryGetValue(TenantHeaderName, out var headerValues) && headerValues.Count > 0)
{
var requestedTenant = headerValues[0]?.Trim();
if (string.IsNullOrEmpty(requestedTenant))
{
problem = Results.Problem(detail: "X-Stella-Tenant header must not be empty.", statusCode: StatusCodes.Status400BadRequest, title: "Validation error");
return false;
}
if (!string.Equals(requestedTenant, options.DefaultTenant, StringComparison.OrdinalIgnoreCase))
{
var detail = string.Format(CultureInfo.InvariantCulture, "Tenant '{0}' is not allowed for this Excititor deployment.", requestedTenant);
problem = Results.Problem(detail: detail, statusCode: StatusCodes.Status403Forbidden, title: "Forbidden");
return false;
}
tenant = requestedTenant;
return true;
}
if (requireHeader)
{
var detail = string.Format(CultureInfo.InvariantCulture, "{0} header is required.", TenantHeaderName);
problem = Results.Problem(detail: detail, statusCode: StatusCodes.Status400BadRequest, title: "Validation error");
return false;
}
return true;
}
private static IReadOnlyDictionary<string, string> ReadMetadata(BsonValue value)
{
if (value is not BsonDocument doc || doc.ElementCount == 0)
{
return new Dictionary<string, string>(StringComparer.Ordinal);
}
var result = new Dictionary<string, string>(StringComparer.Ordinal);
foreach (var element in doc.Elements)
{
if (string.IsNullOrWhiteSpace(element.Name))
{
continue;
}
result[element.Name] = element.Value?.ToString() ?? string.Empty;
}
return result;
}
private static bool TryDecodeCursor(string? cursor, out DateTimeOffset timestamp, out string digest)
{
timestamp = default;
digest = string.Empty;
if (string.IsNullOrWhiteSpace(cursor))
{
return false;
}
try
{
var payload = Encoding.UTF8.GetString(Convert.FromBase64String(cursor));
var parts = payload.Split('|');
if (parts.Length != 2)
{
return false;
}
if (!DateTimeOffset.TryParse(parts[0], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out timestamp))
{
return false;
}
digest = parts[1];
return true;
}
catch
{
return false;
}
}
private static string EncodeCursor(DateTime timestamp, string digest)
{
var payload = string.Format(CultureInfo.InvariantCulture, "{0:O}|{1}", timestamp, digest);
return Convert.ToBase64String(Encoding.UTF8.GetBytes(payload));
}
private static IResult ValidationProblem(string message)
=> Results.Problem(detail: message, statusCode: StatusCodes.Status400BadRequest, title: "Validation error");
private static IResult MapGuardException(ExcititorAocGuardException exception)
{
var violations = exception.Violations.Select(violation => new
{
code = violation.ErrorCode,
path = violation.Path,
message = violation.Message
});
return Results.Problem(
detail: "VEX document failed Aggregation-Only Contract validation.",
statusCode: StatusCodes.Status400BadRequest,
title: "AOC violation",
extensions: new Dictionary<string, object?>
{
["violations"] = violations.ToArray(),
["primaryCode"] = exception.PrimaryErrorCode,
});
}
}

View File

@@ -1,7 +1,10 @@
using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq;
using System.Collections.Immutable;
using System.Globalization;
using System.Text;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Attestation.Verification;
using StellaOps.Excititor.Attestation.Extensions;
@@ -17,13 +20,17 @@ using StellaOps.Excititor.Formats.OpenVEX;
using StellaOps.Excititor.Policy;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.WebService.Endpoints;
using StellaOps.Excititor.WebService.Extensions;
using StellaOps.Excititor.WebService.Options;
using StellaOps.Excititor.WebService.Services;
using StellaOps.Excititor.Core.Aoc;
using StellaOps.Excititor.WebService.Contracts;
using MongoDB.Driver;
using MongoDB.Bson;
var builder = WebApplication.CreateBuilder(args);
var configuration = builder.Configuration;
var services = builder.Services;
var builder = WebApplication.CreateBuilder(args);
var configuration = builder.Configuration;
var services = builder.Services;
services.AddOptions<VexMongoStorageOptions>()
.Bind(configuration.GetSection("Excititor:Storage:Mongo"))
.ValidateOnStart();
@@ -34,8 +41,11 @@ services.AddCycloneDxNormalizer();
services.AddOpenVexNormalizer();
services.AddSingleton<IVexSignatureVerifier, NoopVexSignatureVerifier>();
services.AddScoped<IVexIngestOrchestrator, VexIngestOrchestrator>();
services.AddOptions<ExcititorObservabilityOptions>()
.Bind(configuration.GetSection("Excititor:Observability"));
services.AddScoped<ExcititorHealthService>();
services.AddExcititorAocGuards();
services.AddVexExportEngine();
services.AddVexExportEngine();
services.AddVexExportCacheServices();
services.AddVexAttestation();
services.Configure<VexAttestationClientOptions>(configuration.GetSection("Excititor:Attestation:Client"));
@@ -85,14 +95,17 @@ if (offlineSection.Exists())
services.AddEndpointsApiExplorer();
services.AddHealthChecks();
services.AddSingleton(TimeProvider.System);
services.AddMemoryCache();
services.AddAuthentication();
services.AddAuthorization();
services.AddMemoryCache();
services.AddAuthentication();
services.AddAuthorization();
builder.ConfigureExcititorTelemetry();
var app = builder.Build();
var app = builder.Build();
app.UseAuthentication();
app.UseAuthorization();
app.UseAuthentication();
app.UseAuthorization();
app.UseObservabilityHeaders();
app.MapGet("/excititor/status", async (HttpContext context,
IEnumerable<IVexArtifactStore> artifactStores,
@@ -143,26 +156,428 @@ app.MapGet("/excititor/statements/{vulnerabilityId}/{productKey}", async (
return Results.Ok(claims);
});
app.MapPost("/excititor/admin/backfill-statements", async (
VexStatementBackfillRequest? request,
VexStatementBackfillService backfillService,
CancellationToken cancellationToken) =>
{
app.MapPost("/excititor/admin/backfill-statements", async (
VexStatementBackfillRequest? request,
VexStatementBackfillService backfillService,
CancellationToken cancellationToken) =>
{
request ??= new VexStatementBackfillRequest();
var result = await backfillService.RunAsync(request, cancellationToken).ConfigureAwait(false);
var message = FormattableString.Invariant(
$"Backfill completed: evaluated {result.DocumentsEvaluated}, backfilled {result.DocumentsBackfilled}, claims written {result.ClaimsWritten}, skipped {result.SkippedExisting}, failures {result.NormalizationFailures}.");
return Results.Ok(new
{
message,
summary = result
});
});
IngestEndpoints.MapIngestEndpoints(app);
ResolveEndpoint.MapResolveEndpoint(app);
MirrorEndpoints.MapMirrorEndpoints(app);
return Results.Ok(new
{
message,
summary = result
});
});
app.MapPost("/ingest/vex", async (
HttpContext context,
VexIngestRequest request,
IVexRawStore rawStore,
IOptions<VexMongoStorageOptions> storageOptions,
TimeProvider timeProvider,
ILogger<Program> logger,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.admin");
if (scopeResult is not null)
{
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: true, out var tenant, out var tenantError))
{
return tenantError;
}
VexRawDocument document;
try
{
document = VexRawRequestMapper.Map(request, tenant, timeProvider);
}
catch (Exception ex) when (ex is ArgumentException or InvalidOperationException or FormatException)
{
return ValidationProblem(ex.Message);
}
var existing = await rawStore.FindByDigestAsync(document.Digest, cancellationToken).ConfigureAwait(false);
try
{
await rawStore.StoreAsync(document, cancellationToken).ConfigureAwait(false);
}
catch (ExcititorAocGuardException guardException)
{
logger.LogWarning(
guardException,
"AOC guard rejected VEX ingest tenant={Tenant} digest={Digest}",
tenant,
document.Digest);
return MapGuardException(guardException);
}
var inserted = existing is null;
if (inserted)
{
context.Response.Headers.Location = $"/vex/raw/{Uri.EscapeDataString(document.Digest)}";
}
var response = new VexIngestResponse(document.Digest, inserted, tenant, document.RetrievedAt);
return Results.Json(response, statusCode: inserted ? StatusCodes.Status201Created : StatusCodes.Status200OK);
});
app.MapGet("/vex/raw", async (
HttpContext context,
IMongoDatabase database,
IOptions<VexMongoStorageOptions> storageOptions,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
{
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out _, out var tenantError))
{
return tenantError;
}
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
var query = context.Request.Query;
var filters = new List<FilterDefinition<BsonDocument>>();
var builder = Builders<BsonDocument>.Filter;
if (query.TryGetValue("providerId", out var providerValues))
{
var providers = providerValues
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Select(static value => value!.Trim())
.ToArray();
if (providers.Length > 0)
{
filters.Add(builder.In("ProviderId", providers));
}
}
if (query.TryGetValue("digest", out var digestValues))
{
var digests = digestValues
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Select(static value => value!.Trim())
.ToArray();
if (digests.Length > 0)
{
filters.Add(builder.In("Digest", digests));
}
}
if (query.TryGetValue("format", out var formatValues))
{
var formats = formatValues
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Select(static value => value!.Trim().ToLowerInvariant())
.ToArray();
if (formats.Length > 0)
{
filters.Add(builder.In("Format", formats));
}
}
if (query.TryGetValue("since", out var sinceValues) && DateTimeOffset.TryParse(sinceValues.FirstOrDefault(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var sinceValue))
{
filters.Add(builder.Gte("RetrievedAt", sinceValue.UtcDateTime));
}
var cursorToken = query.TryGetValue("cursor", out var cursorValues) ? cursorValues.FirstOrDefault() : null;
DateTime? cursorTimestamp = null;
string? cursorDigest = null;
if (!string.IsNullOrWhiteSpace(cursorToken) && TryDecodeCursor(cursorToken, out var cursorTime, out var cursorId))
{
cursorTimestamp = cursorTime.UtcDateTime;
cursorDigest = cursorId;
}
if (cursorTimestamp is not null && cursorDigest is not null)
{
var ltTime = builder.Lt("RetrievedAt", cursorTimestamp.Value);
var eqTimeLtDigest = builder.And(
builder.Eq("RetrievedAt", cursorTimestamp.Value),
builder.Lt("Digest", cursorDigest));
filters.Add(builder.Or(ltTime, eqTimeLtDigest));
}
var limit = 50;
if (query.TryGetValue("limit", out var limitValues) && int.TryParse(limitValues.FirstOrDefault(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var requestedLimit))
{
limit = Math.Clamp(requestedLimit, 1, 200);
}
var filter = filters.Count == 0 ? builder.Empty : builder.And(filters);
var sort = Builders<BsonDocument>.Sort.Descending("RetrievedAt").Descending("Digest");
var documents = await collection
.Find(filter)
.Sort(sort)
.Limit(limit)
.Project(Builders<BsonDocument>.Projection.Include("Digest").Include("ProviderId").Include("Format").Include("SourceUri").Include("RetrievedAt").Include("Metadata").Include("GridFsObjectId"))
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var summaries = new List<VexRawSummaryResponse>(documents.Count);
foreach (var document in documents)
{
var digest = document.TryGetValue("Digest", out var digestValue) && digestValue.IsString ? digestValue.AsString : string.Empty;
var providerId = document.TryGetValue("ProviderId", out var providerValue) && providerValue.IsString ? providerValue.AsString : string.Empty;
var format = document.TryGetValue("Format", out var formatValue) && formatValue.IsString ? formatValue.AsString : string.Empty;
var sourceUri = document.TryGetValue("SourceUri", out var sourceValue) && sourceValue.IsString ? sourceValue.AsString : string.Empty;
var retrievedAt = document.TryGetValue("RetrievedAt", out var retrievedValue) && retrievedValue is BsonDateTime bsonDate
? bsonDate.ToUniversalTime()
: DateTime.UtcNow;
var metadata = ReadMetadata(document.TryGetValue("Metadata", out var metadataValue) ? metadataValue : BsonNull.Value);
var inlineContent = !document.TryGetValue("GridFsObjectId", out var gridId) || gridId.IsBsonNull || (gridId.IsString && string.IsNullOrWhiteSpace(gridId.AsString));
summaries.Add(new VexRawSummaryResponse(
digest,
providerId,
format,
sourceUri,
new DateTimeOffset(retrievedAt),
inlineContent,
metadata));
}
var hasMore = documents.Count == limit;
string? nextCursor = null;
if (hasMore && documents.Count > 0)
{
var last = documents[^1];
var lastTime = last.GetValue("RetrievedAt", BsonNull.Value).ToUniversalTime();
var lastDigest = last.GetValue("Digest", BsonNull.Value).AsString;
nextCursor = EncodeCursor(lastTime, lastDigest);
}
return Results.Json(new VexRawListResponse(summaries, nextCursor, hasMore));
});
app.MapGet("/vex/raw/{digest}", async (
string digest,
HttpContext context,
IVexRawStore rawStore,
IOptions<VexMongoStorageOptions> storageOptions,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
{
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out _, out var tenantError))
{
return tenantError;
}
if (string.IsNullOrWhiteSpace(digest))
{
return ValidationProblem("digest is required.");
}
var record = await rawStore.FindByDigestAsync(digest.Trim(), cancellationToken).ConfigureAwait(false);
if (record is null)
{
return Results.NotFound();
}
var rawDocument = VexRawDocumentMapper.ToRawModel(record, storageOptions.Value.DefaultTenant);
var response = new VexRawRecordResponse(record.Digest, rawDocument, record.RetrievedAt);
return Results.Json(response);
});
app.MapGet("/vex/raw/{digest}/provenance", async (
string digest,
HttpContext context,
IVexRawStore rawStore,
IOptions<VexMongoStorageOptions> storageOptions,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
{
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out _, out var tenantError))
{
return tenantError;
}
if (string.IsNullOrWhiteSpace(digest))
{
return ValidationProblem("digest is required.");
}
var record = await rawStore.FindByDigestAsync(digest.Trim(), cancellationToken).ConfigureAwait(false);
if (record is null)
{
return Results.NotFound();
}
var rawDocument = VexRawDocumentMapper.ToRawModel(record, storageOptions.Value.DefaultTenant);
var response = new VexRawProvenanceResponse(
record.Digest,
rawDocument.Tenant,
rawDocument.Source,
rawDocument.Upstream,
record.RetrievedAt);
return Results.Json(response);
});
app.MapPost("/aoc/verify", async (
HttpContext context,
VexAocVerifyRequest? request,
IMongoDatabase database,
IVexRawStore rawStore,
IVexRawWriteGuard guard,
IOptions<VexMongoStorageOptions> storageOptions,
TimeProvider timeProvider,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.admin");
if (scopeResult is not null)
{
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out var tenant, out var tenantError))
{
return tenantError;
}
var now = timeProvider.GetUtcNow();
var since = (request?.Since ?? now.AddHours(-24)).UtcDateTime;
var until = (request?.Until ?? now).UtcDateTime;
if (since >= until)
{
since = until.AddHours(-1);
}
var limit = Math.Clamp(request?.Limit ?? 100, 1, 500);
var sources = request?.Sources?
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Select(static value => value!.Trim())
.ToArray();
var requestedCodes = request?.Codes?
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Select(static value => value!.Trim())
.ToArray();
var builder = Builders<BsonDocument>.Filter;
var filter = builder.And(
builder.Gte("RetrievedAt", since),
builder.Lte("RetrievedAt", until));
if (sources is { Length: > 0 })
{
filter &= builder.In("ProviderId", sources);
}
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
var digests = await collection
.Find(filter)
.Sort(Builders<BsonDocument>.Sort.Descending("RetrievedAt"))
.Limit(limit)
.Project(Builders<BsonDocument>.Projection.Include("Digest").Include("RetrievedAt").Include("ProviderId"))
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var checkedCount = 0;
var violationMap = new Dictionary<string, (int Count, List<VexAocVerifyViolationExample> Examples)>(StringComparer.OrdinalIgnoreCase);
const int MaxExamplesPerCode = 5;
foreach (var digestDocument in digests)
{
var digestValue = digestDocument.GetValue("Digest", BsonNull.Value).AsString;
var provider = digestDocument.GetValue("ProviderId", BsonNull.Value).AsString;
var domainDocument = await rawStore.FindByDigestAsync(digestValue, cancellationToken).ConfigureAwait(false);
if (domainDocument is null)
{
continue;
}
var rawDocument = VexRawDocumentMapper.ToRawModel(domainDocument, storageOptions.Value.DefaultTenant);
try
{
guard.EnsureValid(rawDocument);
checkedCount++;
}
catch (ExcititorAocGuardException guardException)
{
checkedCount++;
foreach (var violation in guardException.Violations)
{
var code = violation.ErrorCode;
if (requestedCodes is { Length: > 0 } && !requestedCodes.Contains(code, StringComparer.OrdinalIgnoreCase))
{
continue;
}
if (!violationMap.TryGetValue(code, out var aggregate))
{
aggregate = (0, new List<VexAocVerifyViolationExample>(MaxExamplesPerCode));
}
aggregate.Count++;
if (aggregate.Examples.Count < MaxExamplesPerCode)
{
aggregate.Examples.Add(new VexAocVerifyViolationExample(
provider,
digestValue,
rawDocument.Upstream.ContentHash,
violation.Path));
}
violationMap[code] = aggregate;
}
}
}
var violations = violationMap
.Select(pair => new VexAocVerifyViolation(pair.Key, pair.Value.Count, pair.Value.Examples))
.OrderByDescending(violation => violation.Count)
.ToList();
var response = new VexAocVerifyResponse(
tenant,
new VexAocVerifyWindow(new DateTimeOffset(since, TimeSpan.Zero), new DateTimeOffset(until, TimeSpan.Zero)),
new VexAocVerifyChecked(0, checkedCount),
violations,
new VexAocVerifyMetrics(checkedCount, violations.Sum(v => v.Count)),
digests.Count == limit);
return Results.Json(response);
});
app.MapGet("/obs/excititor/health", async (
HttpContext httpContext,
ExcititorHealthService healthService,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(httpContext, "vex.admin");
if (scopeResult is not null)
{
return scopeResult;
}
var payload = await healthService.GetAsync(cancellationToken).ConfigureAwait(false);
return Results.Ok(payload);
});
IngestEndpoints.MapIngestEndpoints(app);
ResolveEndpoint.MapResolveEndpoint(app);
MirrorEndpoints.MapMirrorEndpoints(app);
app.Run();

View File

@@ -0,0 +1,667 @@
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using MongoDB.Bson;
using MongoDB.Driver;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.WebService.Options;
namespace StellaOps.Excititor.WebService.Services;
internal sealed class ExcititorHealthService
{
private const string RetrievedAtField = "RetrievedAt";
private const string MetadataField = "Metadata";
private const string CalculatedAtField = "CalculatedAt";
private const string ConflictsField = "Conflicts";
private const string ConflictStatusField = "Status";
private readonly IMongoDatabase _database;
private readonly IVexProviderStore _providerStore;
private readonly IVexConnectorStateRepository _stateRepository;
private readonly IReadOnlyDictionary<string, VexConnectorDescriptor> _connectors;
private readonly TimeProvider _timeProvider;
private readonly ExcititorObservabilityOptions _options;
private readonly ILogger<ExcititorHealthService> _logger;
public ExcititorHealthService(
IMongoDatabase database,
IVexProviderStore providerStore,
IVexConnectorStateRepository stateRepository,
IEnumerable<IVexConnector> connectors,
TimeProvider timeProvider,
IOptions<ExcititorObservabilityOptions> options,
ILogger<ExcititorHealthService> logger)
{
_database = database ?? throw new ArgumentNullException(nameof(database));
_providerStore = providerStore ?? throw new ArgumentNullException(nameof(providerStore));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_timeProvider = timeProvider ?? TimeProvider.System;
_options = options?.Value ?? new ExcititorObservabilityOptions();
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
if (connectors is null)
{
throw new ArgumentNullException(nameof(connectors));
}
_connectors = connectors
.GroupBy(connector => connector.Id, StringComparer.OrdinalIgnoreCase)
.ToDictionary(
group => group.Key,
group => DescribeConnector(group.First()),
StringComparer.OrdinalIgnoreCase);
}
public async Task<ExcititorHealthDocument> GetAsync(CancellationToken cancellationToken)
{
var now = _timeProvider.GetUtcNow();
var providersTask = _providerStore.ListAsync(cancellationToken).AsTask();
var statesTask = _stateRepository.ListAsync(cancellationToken).AsTask();
var signatureTask = LoadSignatureSnapshotAsync(now, cancellationToken);
var conflictTask = LoadConflictSnapshotAsync(now, cancellationToken);
var linkTask = LoadLinkSnapshotAsync(cancellationToken);
await Task.WhenAll(providersTask, statesTask, signatureTask, conflictTask, linkTask).ConfigureAwait(false);
var ingest = BuildIngestSection(now, providersTask.Result, statesTask.Result);
var link = BuildLinkSection(now, linkTask.Result);
var conflicts = BuildConflictSection(conflictTask.Result, link);
var signature = BuildSignatureSection(signatureTask.Result);
return new ExcititorHealthDocument(
now,
ingest,
link,
signature,
conflicts);
}
private IngestHealthSection BuildIngestSection(
DateTimeOffset now,
IReadOnlyCollection<VexProvider> providers,
IReadOnlyCollection<VexConnectorState> states)
{
var providerNames = providers
.GroupBy(provider => provider.Id, StringComparer.OrdinalIgnoreCase)
.ToDictionary(
group => group.Key,
group => group.First().DisplayName,
StringComparer.OrdinalIgnoreCase);
var stateMap = states
.ToDictionary(state => state.ConnectorId, state => state, StringComparer.OrdinalIgnoreCase);
var warningThreshold = _options.GetPositive(_options.IngestWarningThreshold, TimeSpan.FromHours(6));
var criticalThreshold = _options.GetPositive(_options.IngestCriticalThreshold, TimeSpan.FromHours(24));
var connectorHealth = new List<ConnectorHealth>(_connectors.Count);
foreach (var descriptor in _connectors.Values.OrderBy(d => d.Id, StringComparer.OrdinalIgnoreCase))
{
stateMap.TryGetValue(descriptor.Id, out var state);
var displayName = providerNames.TryGetValue(descriptor.Id, out var name)
? name
: descriptor.DisplayName;
var lastSuccess = state?.LastSuccessAt ?? state?.LastUpdated;
double? lagSeconds = null;
if (lastSuccess is not null)
{
var lag = now - lastSuccess.Value;
if (lag < TimeSpan.Zero)
{
lag = TimeSpan.Zero;
}
lagSeconds = lag.TotalSeconds;
}
var status = DetermineIngestStatus(
state?.FailureCount ?? 0,
lagSeconds,
warningThreshold.TotalSeconds,
criticalThreshold.TotalSeconds);
connectorHealth.Add(new ConnectorHealth(
descriptor.Id,
displayName,
status,
lastSuccess,
state?.LastUpdated,
state?.NextEligibleRun,
lagSeconds,
state?.FailureCount ?? 0,
state?.LastFailureReason));
}
var overallStatus = ReduceStatuses(connectorHealth.Select(ch => ch.Status));
double? maxLag = connectorHealth.Any(ch => ch.LagSeconds.HasValue)
? connectorHealth.Max(ch => ch.LagSeconds ?? 0d)
: null;
var maxDetails = _options.MaxConnectorDetails <= 0 ? 50 : _options.MaxConnectorDetails;
var projected = connectorHealth
.OrderByDescending(ch => SeverityRank(ch.Status))
.ThenByDescending(ch => ch.LagSeconds ?? -1)
.ThenBy(ch => ch.ConnectorId, StringComparer.OrdinalIgnoreCase)
.Take(maxDetails)
.ToList();
return new IngestHealthSection(overallStatus, maxLag, projected);
}
private LinkHealthSection BuildLinkSection(DateTimeOffset now, LinkSnapshot snapshot)
{
TimeSpan? lag = null;
if (snapshot.LastConsensusAt is { } calculatedAt)
{
lag = now - calculatedAt;
if (lag < TimeSpan.Zero)
{
lag = TimeSpan.Zero;
}
}
var warning = _options.GetPositive(_options.LinkWarningThreshold, TimeSpan.FromMinutes(15));
var critical = _options.GetPositive(_options.LinkCriticalThreshold, TimeSpan.FromHours(1));
var status = DetermineLagStatus(lag, warning, critical);
return new LinkHealthSection(
status,
snapshot.LastConsensusAt,
lag?.TotalSeconds,
snapshot.TotalDocuments,
snapshot.DocumentsWithConflicts);
}
private ConflictHealthSection BuildConflictSection(ConflictSnapshot snapshot, LinkHealthSection link)
{
var warningRatio = _options.ClampRatio(_options.ConflictWarningRatio, 0.15);
var criticalRatio = _options.ClampRatio(_options.ConflictCriticalRatio, 0.3);
string status;
if (link.TotalDocuments <= 0)
{
status = "unknown";
}
else
{
var ratio = (double)snapshot.DocumentsWithConflicts / link.TotalDocuments;
if (ratio >= criticalRatio)
{
status = "critical";
}
else if (ratio >= warningRatio)
{
status = "warning";
}
else
{
status = "healthy";
}
}
return new ConflictHealthSection(
status,
snapshot.WindowStart,
snapshot.WindowEnd,
snapshot.DocumentsWithConflicts,
snapshot.TotalConflicts,
snapshot.ByStatus,
snapshot.Trend);
}
private SignatureHealthSection BuildSignatureSection(SignatureSnapshot snapshot)
{
if (snapshot.DocumentsEvaluated == 0)
{
return new SignatureHealthSection(
"unknown",
snapshot.WindowStart,
snapshot.WindowEnd,
0,
0,
0,
0,
0,
0);
}
var coverage = snapshot.DocumentsEvaluated == 0
? 0d
: (double)snapshot.Verified / snapshot.DocumentsEvaluated;
var healthy = _options.ClampRatio(_options.SignatureHealthyCoverage, 0.8);
var warning = _options.ClampRatio(_options.SignatureWarningCoverage, 0.5);
if (warning > healthy)
{
warning = healthy;
}
var status = coverage switch
{
var value when value >= healthy => "healthy",
var value when value >= warning => "warning",
_ => "critical"
};
var failures = Math.Max(0, snapshot.WithSignatures - snapshot.Verified);
var unsigned = Math.Max(0, snapshot.DocumentsEvaluated - snapshot.WithSignatures);
return new SignatureHealthSection(
status,
snapshot.WindowStart,
snapshot.WindowEnd,
snapshot.DocumentsEvaluated,
snapshot.WithSignatures,
snapshot.Verified,
failures,
unsigned,
coverage);
}
private async Task<SignatureSnapshot> LoadSignatureSnapshotAsync(DateTimeOffset now, CancellationToken cancellationToken)
{
var window = _options.GetPositive(_options.SignatureWindow, TimeSpan.FromHours(12));
var windowStart = now - window;
var collection = _database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
var filter = Builders<BsonDocument>.Filter.Gte(RetrievedAtField, windowStart.UtcDateTime);
var projection = Builders<BsonDocument>.Projection
.Include(MetadataField)
.Include(RetrievedAtField);
List<BsonDocument> documents;
try
{
documents = await collection
.Find(filter)
.Project(projection)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to load signature window metrics.");
documents = new List<BsonDocument>();
}
var evaluated = 0;
var withSignatures = 0;
var verified = 0;
foreach (var document in documents)
{
evaluated++;
if (!document.TryGetValue(MetadataField, out var metadataValue) ||
metadataValue is not BsonDocument metadata ||
metadata.ElementCount == 0)
{
continue;
}
if (TryGetBoolean(metadata, "signature.present", out var present) && present)
{
withSignatures++;
}
if (TryGetBoolean(metadata, "signature.verified", out var verifiedFlag) && verifiedFlag)
{
verified++;
}
}
return new SignatureSnapshot(windowStart, now, evaluated, withSignatures, verified);
}
private async Task<LinkSnapshot> LoadLinkSnapshotAsync(CancellationToken cancellationToken)
{
var collection = _database.GetCollection<BsonDocument>(VexMongoCollectionNames.Consensus);
BsonDocument? latest = null;
try
{
latest = await collection
.Find(Builders<BsonDocument>.Filter.Empty)
.Sort(Builders<BsonDocument>.Sort.Descending(CalculatedAtField))
.Project(Builders<BsonDocument>.Projection.Include(CalculatedAtField))
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to read latest consensus document.");
}
DateTimeOffset? lastConsensusAt = null;
if (latest is not null &&
latest.TryGetValue(CalculatedAtField, out var dateValue))
{
var utc = TryReadDateTime(dateValue);
if (utc is not null)
{
lastConsensusAt = new DateTimeOffset(utc.Value, TimeSpan.Zero);
}
}
long totalDocuments = 0;
long conflictDocuments = 0;
try
{
totalDocuments = await collection.EstimatedDocumentCountAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
conflictDocuments = await collection.CountDocumentsAsync(
Builders<BsonDocument>.Filter.Exists($"{ConflictsField}.0"),
cancellationToken: cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to compute consensus counts.");
}
return new LinkSnapshot(lastConsensusAt, totalDocuments, conflictDocuments);
}
private async Task<ConflictSnapshot> LoadConflictSnapshotAsync(DateTimeOffset now, CancellationToken cancellationToken)
{
var window = _options.GetPositive(_options.ConflictTrendWindow, TimeSpan.FromHours(24));
var windowStart = now - window;
var collection = _database.GetCollection<BsonDocument>(VexMongoCollectionNames.Consensus);
var filter = Builders<BsonDocument>.Filter.And(
Builders<BsonDocument>.Filter.Gte(CalculatedAtField, windowStart.UtcDateTime),
Builders<BsonDocument>.Filter.Exists($"{ConflictsField}.0"));
var projection = Builders<BsonDocument>.Projection
.Include(CalculatedAtField)
.Include(ConflictsField);
List<BsonDocument> documents;
try
{
documents = await collection
.Find(filter)
.Project(projection)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to load conflict trend window.");
documents = new List<BsonDocument>();
}
var byStatus = new Dictionary<string, long>(StringComparer.OrdinalIgnoreCase);
var timeline = new SortedDictionary<long, long>();
long totalConflicts = 0;
long docsWithConflicts = 0;
var bucketMinutes = Math.Max(1, _options.ConflictTrendBucketMinutes);
var bucketTicks = TimeSpan.FromMinutes(bucketMinutes).Ticks;
foreach (var doc in documents)
{
if (!doc.TryGetValue(ConflictsField, out var conflictsValue) ||
conflictsValue is not BsonArray conflicts ||
conflicts.Count == 0)
{
continue;
}
docsWithConflicts++;
totalConflicts += conflicts.Count;
foreach (var conflictValue in conflicts.OfType<BsonDocument>())
{
var status = conflictValue.TryGetValue(ConflictStatusField, out var statusValue) && statusValue.IsString
? statusValue.AsString
: "unknown";
if (string.IsNullOrWhiteSpace(status))
{
status = "unknown";
}
byStatus[status] = byStatus.TryGetValue(status, out var current)
? current + 1
: 1;
}
if (doc.TryGetValue(CalculatedAtField, out var calculatedValue))
{
var utc = TryReadDateTime(calculatedValue);
if (utc is null)
{
continue;
}
var alignedTicks = AlignTicks(utc.Value, bucketTicks);
timeline[alignedTicks] = timeline.TryGetValue(alignedTicks, out var current)
? current + conflicts.Count
: conflicts.Count;
}
}
var trend = timeline
.Select(pair => new ConflictTrendPoint(
new DateTimeOffset(pair.Key, TimeSpan.Zero),
pair.Value))
.ToList();
return new ConflictSnapshot(
windowStart,
now,
docsWithConflicts,
totalConflicts,
new Dictionary<string, long>(byStatus, StringComparer.OrdinalIgnoreCase),
trend);
}
private static string DetermineIngestStatus(int failureCount, double? lagSeconds, double warningSeconds, double criticalSeconds)
{
if (failureCount > 0)
{
return "failing";
}
if (lagSeconds is null)
{
return "unknown";
}
if (lagSeconds.Value >= criticalSeconds)
{
return "critical";
}
if (lagSeconds.Value >= warningSeconds)
{
return "warning";
}
return "healthy";
}
private static string DetermineLagStatus(TimeSpan? lag, TimeSpan warning, TimeSpan critical)
{
if (lag is null)
{
return "unknown";
}
if (lag.Value >= critical)
{
return "critical";
}
if (lag.Value >= warning)
{
return "warning";
}
return "healthy";
}
private static string ReduceStatuses(IEnumerable<string> statuses)
{
var highest = "unknown";
var highestRank = -1;
foreach (var status in statuses)
{
var rank = SeverityRank(status);
if (rank > highestRank)
{
highestRank = rank;
highest = status;
}
}
return highest;
}
private static int SeverityRank(string? status)
=> status?.ToLowerInvariant() switch
{
"critical" => 4,
"failing" => 3,
"warning" => 2,
"unknown" => 1,
_ => 0,
};
private static long AlignTicks(DateTime dateTimeUtc, long bucketTicks)
{
var ticks = dateTimeUtc.Ticks;
return ticks - (ticks % bucketTicks);
}
private static DateTime? TryReadDateTime(BsonValue value)
{
if (value is null)
{
return null;
}
if (value.IsBsonDateTime)
{
return value.AsBsonDateTime.ToUniversalTime();
}
if (value.IsString &&
DateTime.TryParse(
value.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal,
out var parsed))
{
return DateTime.SpecifyKind(parsed, DateTimeKind.Utc);
}
return null;
}
private static bool TryGetBoolean(BsonDocument document, string key, out bool value)
{
value = default;
if (!document.TryGetValue(key, out var bsonValue))
{
return false;
}
if (bsonValue.IsBoolean)
{
value = bsonValue.AsBoolean;
return true;
}
if (bsonValue.IsString && bool.TryParse(bsonValue.AsString, out var parsed))
{
value = parsed;
return true;
}
return false;
}
private static VexConnectorDescriptor DescribeConnector(IVexConnector connector)
=> connector switch
{
VexConnectorBase baseConnector => baseConnector.Descriptor,
_ => new VexConnectorDescriptor(connector.Id, connector.Kind, connector.Id)
};
private sealed record LinkSnapshot(DateTimeOffset? LastConsensusAt, long TotalDocuments, long DocumentsWithConflicts);
private sealed record ConflictSnapshot(
DateTimeOffset WindowStart,
DateTimeOffset WindowEnd,
long DocumentsWithConflicts,
long TotalConflicts,
IReadOnlyDictionary<string, long> ByStatus,
IReadOnlyList<ConflictTrendPoint> Trend);
private sealed record SignatureSnapshot(
DateTimeOffset WindowStart,
DateTimeOffset WindowEnd,
int DocumentsEvaluated,
int WithSignatures,
int Verified);
}
internal sealed record ExcititorHealthDocument(
DateTimeOffset GeneratedAt,
IngestHealthSection Ingest,
LinkHealthSection Link,
SignatureHealthSection Signature,
ConflictHealthSection Conflicts);
internal sealed record IngestHealthSection(
string Status,
double? MaxLagSeconds,
IReadOnlyList<ConnectorHealth> Connectors);
internal sealed record ConnectorHealth(
string ConnectorId,
string DisplayName,
string Status,
DateTimeOffset? LastSuccessAt,
DateTimeOffset? LastUpdated,
DateTimeOffset? NextEligibleRun,
double? LagSeconds,
int FailureCount,
string? LastFailureReason);
internal sealed record LinkHealthSection(
string Status,
DateTimeOffset? LastConsensusAt,
double? LagSeconds,
long TotalDocuments,
long DocumentsWithConflicts);
internal sealed record SignatureHealthSection(
string Status,
DateTimeOffset WindowStart,
DateTimeOffset WindowEnd,
int DocumentsEvaluated,
int WithSignatures,
int Verified,
int Failures,
int Unsigned,
double Coverage);
internal sealed record ConflictHealthSection(
string Status,
DateTimeOffset WindowStart,
DateTimeOffset WindowEnd,
long DocumentsWithConflicts,
long ConflictStatements,
IReadOnlyDictionary<string, long> ByStatus,
IReadOnlyList<ConflictTrendPoint> Trend);
internal sealed record ConflictTrendPoint(DateTimeOffset BucketStart, long Conflicts);

View File

@@ -1,9 +1,11 @@
using System.Collections.Immutable;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
@@ -21,43 +23,50 @@ internal interface IVexIngestOrchestrator
Task<ReconcileSummary> ReconcileAsync(ReconcileOptions options, CancellationToken cancellationToken);
}
internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
{
private readonly IServiceProvider _serviceProvider;
private readonly IReadOnlyDictionary<string, IVexConnector> _connectors;
private readonly IVexRawStore _rawStore;
private readonly IVexClaimStore _claimStore;
private readonly IVexProviderStore _providerStore;
private readonly IVexConnectorStateRepository _stateRepository;
private readonly IVexNormalizerRouter _normalizerRouter;
private readonly IVexSignatureVerifier _signatureVerifier;
private readonly IVexMongoSessionProvider _sessionProvider;
private readonly TimeProvider _timeProvider;
private readonly ILogger<VexIngestOrchestrator> _logger;
internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
{
private readonly IServiceProvider _serviceProvider;
private readonly IReadOnlyDictionary<string, IVexConnector> _connectors;
private readonly IVexRawStore _rawStore;
private readonly IVexClaimStore _claimStore;
private readonly IVexProviderStore _providerStore;
private readonly IVexConnectorStateRepository _stateRepository;
private readonly IVexNormalizerRouter _normalizerRouter;
private readonly IVexSignatureVerifier _signatureVerifier;
private readonly IVexMongoSessionProvider _sessionProvider;
private readonly TimeProvider _timeProvider;
private readonly ILogger<VexIngestOrchestrator> _logger;
private readonly string _defaultTenant;
public VexIngestOrchestrator(
IServiceProvider serviceProvider,
IEnumerable<IVexConnector> connectors,
IVexRawStore rawStore,
IVexClaimStore claimStore,
IVexProviderStore providerStore,
IVexConnectorStateRepository stateRepository,
IVexNormalizerRouter normalizerRouter,
IVexSignatureVerifier signatureVerifier,
IVexMongoSessionProvider sessionProvider,
TimeProvider timeProvider,
ILogger<VexIngestOrchestrator> logger)
{
_serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
_rawStore = rawStore ?? throw new ArgumentNullException(nameof(rawStore));
_claimStore = claimStore ?? throw new ArgumentNullException(nameof(claimStore));
public VexIngestOrchestrator(
IServiceProvider serviceProvider,
IEnumerable<IVexConnector> connectors,
IVexRawStore rawStore,
IVexClaimStore claimStore,
IVexProviderStore providerStore,
IVexConnectorStateRepository stateRepository,
IVexNormalizerRouter normalizerRouter,
IVexSignatureVerifier signatureVerifier,
IVexMongoSessionProvider sessionProvider,
TimeProvider timeProvider,
IOptions<VexMongoStorageOptions> storageOptions,
ILogger<VexIngestOrchestrator> logger)
{
_serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
_rawStore = rawStore ?? throw new ArgumentNullException(nameof(rawStore));
_claimStore = claimStore ?? throw new ArgumentNullException(nameof(claimStore));
_providerStore = providerStore ?? throw new ArgumentNullException(nameof(providerStore));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_normalizerRouter = normalizerRouter ?? throw new ArgumentNullException(nameof(normalizerRouter));
_signatureVerifier = signatureVerifier ?? throw new ArgumentNullException(nameof(signatureVerifier));
_sessionProvider = sessionProvider ?? throw new ArgumentNullException(nameof(sessionProvider));
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_sessionProvider = sessionProvider ?? throw new ArgumentNullException(nameof(sessionProvider));
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
var optionsValue = (storageOptions ?? throw new ArgumentNullException(nameof(storageOptions))).Value
?? throw new ArgumentNullException(nameof(storageOptions));
_defaultTenant = string.IsNullOrWhiteSpace(optionsValue.DefaultTenant)
? "default"
: optionsValue.DefaultTenant.Trim();
if (connectors is null)
{
@@ -149,7 +158,7 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
foreach (var handle in handles)
{
var result = await ExecuteRunAsync(handle, since, options.Force, session, cancellationToken).ConfigureAwait(false);
var result = await ExecuteRunAsync(runId, handle, since, options.Force, session, cancellationToken).ConfigureAwait(false);
results.Add(result);
}
@@ -172,12 +181,12 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
results.Add(ProviderRunResult.Missing(providerId, since: null));
}
foreach (var handle in handles)
{
var since = await ResolveResumeSinceAsync(handle.Descriptor.Id, options.Checkpoint, session, cancellationToken).ConfigureAwait(false);
var result = await ExecuteRunAsync(handle, since, force: false, session, cancellationToken).ConfigureAwait(false);
results.Add(result);
}
foreach (var handle in handles)
{
var since = await ResolveResumeSinceAsync(handle.Descriptor.Id, options.Checkpoint, session, cancellationToken).ConfigureAwait(false);
var result = await ExecuteRunAsync(runId, handle, since, force: false, session, cancellationToken).ConfigureAwait(false);
results.Add(result);
}
var completedAt = _timeProvider.GetUtcNow();
return new IngestRunSummary(runId, startedAt, completedAt, results.ToImmutable());
@@ -210,8 +219,8 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
if (stale || state is null)
{
var since = stale ? threshold : lastUpdated;
var result = await ExecuteRunAsync(handle, since, force: false, session, cancellationToken).ConfigureAwait(false);
results.Add(new ReconcileProviderResult(
var result = await ExecuteRunAsync(runId, handle, since, force: false, session, cancellationToken).ConfigureAwait(false);
results.Add(new ReconcileProviderResult(
handle.Descriptor.Id,
result.Status,
"reconciled",
@@ -283,16 +292,25 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
await _providerStore.SaveAsync(provider, cancellationToken, session).ConfigureAwait(false);
}
private async Task<ProviderRunResult> ExecuteRunAsync(
ConnectorHandle handle,
DateTimeOffset? since,
bool force,
IClientSessionHandle session,
CancellationToken cancellationToken)
{
var providerId = handle.Descriptor.Id;
var startedAt = _timeProvider.GetUtcNow();
var stopwatch = Stopwatch.StartNew();
private async Task<ProviderRunResult> ExecuteRunAsync(
Guid runId,
ConnectorHandle handle,
DateTimeOffset? since,
bool force,
IClientSessionHandle session,
CancellationToken cancellationToken)
{
var providerId = handle.Descriptor.Id;
var startedAt = _timeProvider.GetUtcNow();
var stopwatch = Stopwatch.StartNew();
using var scope = _logger.BeginScope(new Dictionary<string, object?>(StringComparer.Ordinal)
{
["tenant"] = _defaultTenant,
["runId"] = runId,
["providerId"] = providerId,
["window.since"] = since?.ToString("O", CultureInfo.InvariantCulture),
["force"] = force,
});
try
{

View File

@@ -7,6 +7,14 @@
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.12.0" />
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.12.0" />
<PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.12.0" />
<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.12.0" />
<PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.12.0" />
<PackageReference Include="OpenTelemetry.Instrumentation.Runtime" Version="1.12.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj" />
@@ -18,6 +26,7 @@
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Formats.CSAF/StellaOps.Excititor.Formats.CSAF.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Formats.CycloneDX/StellaOps.Excititor.Formats.CycloneDX.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Formats.OpenVEX/StellaOps.Excititor.Formats.OpenVEX.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Ingestion.Telemetry/StellaOps.Ingestion.Telemetry.csproj" />
<ProjectReference Include="../../Aoc/__Libraries/StellaOps.Aoc/StellaOps.Aoc.csproj" />
</ItemGroup>
</Project>
</Project>

View File

@@ -1,94 +1,95 @@
# TASKS — Epic 1: Aggregation-Only Contract
> **AOC Reminder:** Excititor WebService publishes raw statements/linksets only; derived precedence/severity belongs to Policy overlays.
| ID | Status | Owner(s) | Depends on | Notes |
|---|---|---|---|---|
| EXCITITOR-WEB-AOC-19-001 `Raw VEX ingestion APIs` | TODO | Excititor WebService Guild | EXCITITOR-CORE-AOC-19-001, EXCITITOR-STORE-AOC-19-001 | Implement `POST /ingest/vex`, `GET /vex/raw*`, and `POST /aoc/verify` endpoints. Enforce Authority scopes, tenant injection, and guard pipeline to ensure only immutable VEX facts are persisted. |
> Docs alignment (2025-10-26): See AOC reference §45 and authority scopes doc for required tokens/behaviour.
| EXCITITOR-WEB-AOC-19-002 `AOC observability + metrics` | TODO | Excititor WebService Guild, Observability Guild | EXCITITOR-WEB-AOC-19-001 | Export metrics (`ingestion_write_total`, `aoc_violation_total`, signature verification counters) and tracing spans matching Conseiller naming. Ensure structured logging includes tenant, source vendor, upstream id, and content hash. |
> Docs alignment (2025-10-26): Metrics/traces/log schema in `docs/observability/observability.md`.
| EXCITITOR-WEB-AOC-19-003 `Guard + schema test harness` | TODO | QA Guild | EXCITITOR-WEB-AOC-19-001 | Add unit/integration tests for schema validation, forbidden field rejection (`ERR_AOC_001/006/007`), and supersedes behavior using CycloneDX-VEX & CSAF fixtures with deterministic expectations. |
> Docs alignment (2025-10-26): Error codes + CLI verification in `docs/modules/cli/guides/cli-reference.md`.
| EXCITITOR-WEB-AOC-19-004 `Batch ingest validation` | TODO | Excititor WebService Guild, QA Guild | EXCITITOR-WEB-AOC-19-003, EXCITITOR-CORE-AOC-19-002 | Build large fixture ingest covering mixed VEX statuses, verifying raw storage parity, metrics, and CLI `aoc verify` compatibility. Document load test/runbook updates. |
> Docs alignment (2025-10-26): Offline/air-gap workflows captured in `docs/deploy/containers.md` §5.
## Policy Engine v2
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-POLICY-20-001 `Policy selection endpoints` | TODO | Excititor WebService Guild | WEB-POLICY-20-001, EXCITITOR-CORE-AOC-19-004 | Provide VEX lookup APIs supporting PURL/advisory batching, scope filtering, and tenant enforcement with deterministic ordering + pagination. |
## StellaOps Console (Sprint 23)
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-CONSOLE-23-001 `VEX aggregation views` | TODO | Excititor WebService Guild, BE-Base Platform Guild | EXCITITOR-LNM-21-201, EXCITITOR-LNM-21-202 | Expose `/console/vex` endpoints returning grouped VEX statements per advisory/component with status chips, justification metadata, precedence trace pointers, and tenant-scoped filters for Console explorer. |
| EXCITITOR-CONSOLE-23-002 `Dashboard VEX deltas` | TODO | Excititor WebService Guild | EXCITITOR-CONSOLE-23-001, EXCITITOR-LNM-21-203 | Provide aggregated counts for VEX overrides (new, not_affected, revoked) powering Console dashboard + live status ticker; emit metrics for policy explain integration. |
| EXCITITOR-CONSOLE-23-003 `VEX search helpers` | TODO | Excititor WebService Guild | EXCITITOR-CONSOLE-23-001 | Deliver rapid lookup endpoints of VEX by advisory/component for Console global search; ensure response includes provenance and precedence context; include caching and RBAC. |
## Graph Explorer v1
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
## Link-Not-Merge v1
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-LNM-21-201 `Observation APIs` | TODO | Excititor WebService Guild, BE-Base Platform Guild | EXCITITOR-LNM-21-001 | Add VEX observation read endpoints with filters, pagination, RBAC, and tenant scoping. |
| EXCITITOR-LNM-21-202 `Linkset APIs` | TODO | Excititor WebService Guild | EXCITITOR-LNM-21-002, EXCITITOR-LNM-21-003 | Implement linkset read/export/evidence endpoints returning correlation/conflict payloads and map errors to `ERR_AGG_*`. |
| EXCITITOR-LNM-21-203 `Event publishing` | TODO | Excititor WebService Guild, Platform Events Guild | EXCITITOR-LNM-21-005 | Publish `vex.linkset.updated` events, document schema, and ensure idempotent delivery. |
## Graph & Vuln Explorer v1
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-GRAPH-24-101 `VEX summary API` | TODO | Excititor WebService Guild | EXCITITOR-GRAPH-24-001 | Provide endpoints delivering VEX status summaries per component/asset for Vuln Explorer integration. |
| EXCITITOR-GRAPH-24-102 `Evidence batch API` | TODO | Excititor WebService Guild | EXCITITOR-LNM-21-201 | Add batch VEX observation retrieval optimized for Graph overlays/tooltips. |
## VEX Lens (Sprint 30)
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-VEXLENS-30-001 `VEX evidence enrichers` | TODO | Excititor WebService Guild, VEX Lens Guild | EXCITITOR-VULN-29-001, VEXLENS-30-005 | Include issuer hints, signatures, and product trees in evidence payloads for VEX Lens; Label: VEX-Lens. |
## Vulnerability Explorer (Sprint 29)
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-VULN-29-001 `VEX key canonicalization` | TODO | Excititor WebService Guild | EXCITITOR-LNM-21-001 | Canonicalize (lossless) VEX advisory/product keys (map to `advisory_key`, capture product scopes); expose original sources in `links[]`; AOC-compliant: no merge, no derived fields, no suppression; backfill existing records. |
| EXCITITOR-VULN-29-002 `Evidence retrieval` | TODO | Excititor WebService Guild | EXCITITOR-VULN-29-001, VULN-API-29-003 | Provide `/vuln/evidence/vex/{advisory_key}` returning raw VEX statements filtered by tenant/product scope for Explorer evidence tabs. |
| EXCITITOR-VULN-29-004 `Observability` | TODO | Excititor WebService Guild, Observability Guild | EXCITITOR-VULN-29-001 | Add metrics/logs for VEX normalization, suppression scopes, withdrawn statements; emit events consumed by Vuln Explorer resolver. |
## Advisory AI (Sprint 31)
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-AIAI-31-001 `Justification enrichment` | TODO | Excititor WebService Guild | EXCITITOR-VULN-29-001 | Expose normalized VEX justifications, product trees, and paragraph anchors for Advisory AI conflict explanations. |
| EXCITITOR-AIAI-31-002 `VEX chunk API` | TODO | Excititor WebService Guild | EXCITITOR-AIAI-31-001, VEXLENS-30-006 | Provide `/vex/evidence/chunks` endpoint returning tenant-scoped VEX statements with signature metadata and scope scores for RAG. |
| EXCITITOR-AIAI-31-003 `Telemetry` | TODO | Excititor WebService Guild, Observability Guild | EXCITITOR-AIAI-31-001 | Emit metrics/logs for VEX chunk usage, signature verification failures, and guardrail triggers. |
## Observability & Forensics (Epic 15)
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-WEB-OBS-50-001 `Telemetry adoption` | TODO | Excititor WebService Guild | TELEMETRY-OBS-50-001, EXCITITOR-OBS-50-001 | Adopt telemetry core for VEX APIs, ensure responses include trace IDs & correlation headers, and update structured logging for read endpoints. |
| EXCITITOR-WEB-OBS-51-001 `Observability health endpoints` | TODO | Excititor WebService Guild | EXCITITOR-WEB-OBS-50-001, WEB-OBS-51-001 | Implement `/obs/excititor/health` summarizing ingest/link SLOs, signature failure counts, and conflict trends for Console dashboards. |
| EXCITITOR-WEB-OBS-52-001 `Timeline streaming` | TODO | Excititor WebService Guild | EXCITITOR-WEB-OBS-50-001, TIMELINE-OBS-52-003 | Provide SSE bridge for VEX timeline events with tenant filters, pagination, and guardrails. |
| EXCITITOR-WEB-OBS-53-001 `Evidence APIs` | TODO | Excititor WebService Guild, Evidence Locker Guild | EXCITITOR-OBS-53-001, EVID-OBS-53-003 | Expose `/evidence/vex/*` endpoints that fetch locker bundles, enforce scopes, and surface verification metadata. |
| EXCITITOR-WEB-OBS-54-001 `Attestation APIs` | TODO | Excititor WebService Guild | EXCITITOR-OBS-54-001, PROV-OBS-54-001 | Add `/attestations/vex/*` endpoints returning DSSE verification state, builder identity, and chain-of-custody links. |
| EXCITITOR-WEB-OBS-55-001 `Incident mode toggles` | TODO | Excititor WebService Guild, DevOps Guild | EXCITITOR-OBS-55-001, WEB-OBS-55-001 | Provide incident mode API for VEX pipelines with activation audit logs and retention override previews. |
## Air-Gapped Mode (Epic 16)
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-WEB-AIRGAP-56-001 | TODO | Excititor WebService Guild | AIRGAP-IMP-58-001, EXCITITOR-AIRGAP-56-001 | Support mirror bundle registration via APIs, expose bundle provenance in VEX responses, and block external connectors in sealed mode. |
| EXCITITOR-WEB-AIRGAP-56-002 | TODO | Excititor WebService Guild, AirGap Time Guild | EXCITITOR-WEB-AIRGAP-56-001, AIRGAP-TIME-58-001 | Return VEX staleness metrics and time anchor info in API responses for Console/CLI use. |
| EXCITITOR-WEB-AIRGAP-57-001 | TODO | Excititor WebService Guild, AirGap Policy Guild | AIRGAP-POL-56-001 | Map sealed-mode violations to standardized error payload with remediation guidance. |
| EXCITITOR-WEB-AIRGAP-58-001 | TODO | Excititor WebService Guild, AirGap Importer Guild | EXCITITOR-WEB-AIRGAP-56-001, TIMELINE-OBS-53-001 | Emit timeline events for VEX bundle imports with bundle ID, scope, and actor metadata. |
## SDKs & OpenAPI (Epic 17)
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-WEB-OAS-61-001 | TODO | Excititor WebService Guild | OAS-61-001 | Implement `/.well-known/openapi` discovery endpoint with spec version metadata. |
| EXCITITOR-WEB-OAS-61-002 | TODO | Excititor WebService Guild | APIGOV-61-001 | Standardize error envelope responses and update controller/unit tests. |
| EXCITITOR-WEB-OAS-62-001 | TODO | Excititor WebService Guild | EXCITITOR-OAS-61-002 | Add curated examples for VEX observation/linkset endpoints and ensure portal displays them. |
| EXCITITOR-WEB-OAS-63-001 | TODO | Excititor WebService Guild, API Governance Guild | APIGOV-63-001 | Emit deprecation headers and update docs for retiring VEX APIs. |
# TASKS — Epic 1: Aggregation-Only Contract
> **AOC Reminder:** Excititor WebService publishes raw statements/linksets only; derived precedence/severity belongs to Policy overlays.
| ID | Status | Owner(s) | Depends on | Notes |
|---|---|---|---|---|
| EXCITITOR-WEB-AOC-19-001 `Raw VEX ingestion APIs` | DONE (2025-11-08) | Excititor WebService Guild | EXCITITOR-CORE-AOC-19-001, EXCITITOR-STORE-AOC-19-001 | Implement `POST /ingest/vex`, `GET /vex/raw*`, and `POST /aoc/verify` endpoints. Enforce Authority scopes, tenant injection, and guard pipeline to ensure only immutable VEX facts are persisted. |
> Docs alignment (2025-10-26): See AOC reference §45 and authority scopes doc for required tokens/behaviour.
| EXCITITOR-WEB-AOC-19-002 `AOC observability + metrics` | DONE (2025-11-08) | Excititor WebService Guild, Observability Guild | EXCITITOR-WEB-AOC-19-001 | Export metrics (`ingestion_write_total`, `aoc_violation_total`, signature verification counters) and tracing spans matching Conseiller naming. Ensure structured logging includes tenant, source vendor, upstream id, and content hash. |
> Docs alignment (2025-10-26): Metrics/traces/log schema in `docs/observability/observability.md`.
| EXCITITOR-WEB-AOC-19-003 `Guard + schema test harness` | DONE (2025-11-08) | QA Guild | EXCITITOR-WEB-AOC-19-001 | Add unit/integration tests for schema validation, forbidden field rejection (`ERR_AOC_001/006/007`), and supersedes behavior using CycloneDX-VEX & CSAF fixtures with deterministic expectations. |
> Docs alignment (2025-10-26): Error codes + CLI verification in `docs/modules/cli/guides/cli-reference.md`.
| EXCITITOR-WEB-AOC-19-004 `Batch ingest validation` | DONE (2025-11-08) | Excititor WebService Guild, QA Guild | EXCITITOR-WEB-AOC-19-003, EXCITITOR-CORE-AOC-19-002 | Build large fixture ingest covering mixed VEX statuses, verifying raw storage parity, metrics, and CLI `aoc verify` compatibility. Document load test/runbook updates. |
> Docs alignment (2025-10-26): Offline/air-gap workflows captured in `docs/deploy/containers.md` §5.
| EXCITITOR-CRYPTO-90-001 `Crypto provider adoption` | TODO | Excititor WebService Guild, Security Guild | SEC-CRYPTO-90-003, SEC-CRYPTO-90-004 | Replace direct `System.Security.Cryptography` hashing/signing inside connector loaders, VEX exporters, and OpenAPI discovery with `ICryptoProviderRegistry` + `ICryptoHash`. Reference `docs/security/crypto-routing-audit-2025-11-07.md`. | Registry-backed providers configurable per deployment; integration tests cover default + `ru-offline` profiles; connectors honor sovereign provider ordering. |
## Policy Engine v2
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-POLICY-20-001 `Policy selection endpoints` | TODO | Excititor WebService Guild | WEB-POLICY-20-001, EXCITITOR-CORE-AOC-19-004 | Provide VEX lookup APIs supporting PURL/advisory batching, scope filtering, and tenant enforcement with deterministic ordering + pagination. |
## StellaOps Console (Sprint 23)
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-CONSOLE-23-001 `VEX aggregation views` | TODO | Excititor WebService Guild, BE-Base Platform Guild | EXCITITOR-LNM-21-201, EXCITITOR-LNM-21-202 | Expose `/console/vex` endpoints returning grouped VEX statements per advisory/component with status chips, justification metadata, precedence trace pointers, and tenant-scoped filters for Console explorer. |
| EXCITITOR-CONSOLE-23-002 `Dashboard VEX deltas` | TODO | Excititor WebService Guild | EXCITITOR-CONSOLE-23-001, EXCITITOR-LNM-21-203 | Provide aggregated counts for VEX overrides (new, not_affected, revoked) powering Console dashboard + live status ticker; emit metrics for policy explain integration. |
| EXCITITOR-CONSOLE-23-003 `VEX search helpers` | TODO | Excititor WebService Guild | EXCITITOR-CONSOLE-23-001 | Deliver rapid lookup endpoints of VEX by advisory/component for Console global search; ensure response includes provenance and precedence context; include caching and RBAC. |
## Graph Explorer v1
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
## Link-Not-Merge v1
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-LNM-21-201 `Observation APIs` | TODO | Excititor WebService Guild, BE-Base Platform Guild | EXCITITOR-LNM-21-001 | Add VEX observation read endpoints with filters, pagination, RBAC, and tenant scoping. |
| EXCITITOR-LNM-21-202 `Linkset APIs` | TODO | Excititor WebService Guild | EXCITITOR-LNM-21-002, EXCITITOR-LNM-21-003 | Implement linkset read/export/evidence endpoints returning correlation/conflict payloads and map errors to `ERR_AGG_*`. |
| EXCITITOR-LNM-21-203 `Event publishing` | TODO | Excititor WebService Guild, Platform Events Guild | EXCITITOR-LNM-21-005 | Publish `vex.linkset.updated` events, document schema, and ensure idempotent delivery. |
## Graph & Vuln Explorer v1
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-GRAPH-24-101 `VEX summary API` | TODO | Excititor WebService Guild | EXCITITOR-GRAPH-24-001 | Provide endpoints delivering VEX status summaries per component/asset for Vuln Explorer integration. |
| EXCITITOR-GRAPH-24-102 `Evidence batch API` | TODO | Excititor WebService Guild | EXCITITOR-LNM-21-201 | Add batch VEX observation retrieval optimized for Graph overlays/tooltips. |
## VEX Lens (Sprint 30)
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-VEXLENS-30-001 `VEX evidence enrichers` | TODO | Excititor WebService Guild, VEX Lens Guild | EXCITITOR-VULN-29-001, VEXLENS-30-005 | Include issuer hints, signatures, and product trees in evidence payloads for VEX Lens; Label: VEX-Lens. |
## Vulnerability Explorer (Sprint 29)
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-VULN-29-001 `VEX key canonicalization` | TODO | Excititor WebService Guild | EXCITITOR-LNM-21-001 | Canonicalize (lossless) VEX advisory/product keys (map to `advisory_key`, capture product scopes); expose original sources in `links[]`; AOC-compliant: no merge, no derived fields, no suppression; backfill existing records. |
| EXCITITOR-VULN-29-002 `Evidence retrieval` | TODO | Excititor WebService Guild | EXCITITOR-VULN-29-001, VULN-API-29-003 | Provide `/vuln/evidence/vex/{advisory_key}` returning raw VEX statements filtered by tenant/product scope for Explorer evidence tabs. |
| EXCITITOR-VULN-29-004 `Observability` | TODO | Excititor WebService Guild, Observability Guild | EXCITITOR-VULN-29-001 | Add metrics/logs for VEX normalization, suppression scopes, withdrawn statements; emit events consumed by Vuln Explorer resolver. |
## Advisory AI (Sprint 31)
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-AIAI-31-001 `Justification enrichment` | TODO | Excititor WebService Guild | EXCITITOR-VULN-29-001 | Expose normalized VEX justifications, product trees, and paragraph anchors for Advisory AI conflict explanations. |
| EXCITITOR-AIAI-31-002 `VEX chunk API` | TODO | Excititor WebService Guild | EXCITITOR-AIAI-31-001, VEXLENS-30-006 | Provide `/vex/evidence/chunks` endpoint returning tenant-scoped VEX statements with signature metadata and scope scores for RAG. |
| EXCITITOR-AIAI-31-003 `Telemetry` | TODO | Excititor WebService Guild, Observability Guild | EXCITITOR-AIAI-31-001 | Emit metrics/logs for VEX chunk usage, signature verification failures, and guardrail triggers. |
## Observability & Forensics (Epic 15)
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-WEB-OBS-50-001 `Telemetry adoption` | DONE (2025-11-07) | Excititor WebService Guild | TELEMETRY-OBS-50-001, EXCITITOR-OBS-50-001 | Adopt telemetry core for VEX APIs, ensure responses include trace IDs & correlation headers, and update structured logging for read endpoints. |
| EXCITITOR-WEB-OBS-51-001 `Observability health endpoints` | DONE (2025-11-08) | Excititor WebService Guild | EXCITITOR-WEB-OBS-50-001, WEB-OBS-51-001 | Implement `/obs/excititor/health` summarizing ingest/link SLOs, signature failure counts, and conflict trends for Console dashboards. |
| EXCITITOR-WEB-OBS-52-001 `Timeline streaming` | TODO | Excititor WebService Guild | EXCITITOR-WEB-OBS-50-001, TIMELINE-OBS-52-003 | Provide SSE bridge for VEX timeline events with tenant filters, pagination, and guardrails. |
| EXCITITOR-WEB-OBS-53-001 `Evidence APIs` | TODO | Excititor WebService Guild, Evidence Locker Guild | EXCITITOR-OBS-53-001, EVID-OBS-53-003 | Expose `/evidence/vex/*` endpoints that fetch locker bundles, enforce scopes, and surface verification metadata. |
| EXCITITOR-WEB-OBS-54-001 `Attestation APIs` | TODO | Excititor WebService Guild | EXCITITOR-OBS-54-001, PROV-OBS-54-001 | Add `/attestations/vex/*` endpoints returning DSSE verification state, builder identity, and chain-of-custody links. |
| EXCITITOR-WEB-OBS-55-001 `Incident mode toggles` | TODO | Excititor WebService Guild, DevOps Guild | EXCITITOR-OBS-55-001, WEB-OBS-55-001 | Provide incident mode API for VEX pipelines with activation audit logs and retention override previews. |
## Air-Gapped Mode (Epic 16)
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-WEB-AIRGAP-56-001 | TODO | Excititor WebService Guild | AIRGAP-IMP-58-001, EXCITITOR-AIRGAP-56-001 | Support mirror bundle registration via APIs, expose bundle provenance in VEX responses, and block external connectors in sealed mode. |
| EXCITITOR-WEB-AIRGAP-56-002 | TODO | Excititor WebService Guild, AirGap Time Guild | EXCITITOR-WEB-AIRGAP-56-001, AIRGAP-TIME-58-001 | Return VEX staleness metrics and time anchor info in API responses for Console/CLI use. |
| EXCITITOR-WEB-AIRGAP-57-001 | TODO | Excititor WebService Guild, AirGap Policy Guild | AIRGAP-POL-56-001 | Map sealed-mode violations to standardized error payload with remediation guidance. |
| EXCITITOR-WEB-AIRGAP-58-001 | TODO | Excititor WebService Guild, AirGap Importer Guild | EXCITITOR-WEB-AIRGAP-56-001, TIMELINE-OBS-53-001 | Emit timeline events for VEX bundle imports with bundle ID, scope, and actor metadata. |
## SDKs & OpenAPI (Epic 17)
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-WEB-OAS-61-001 | TODO | Excititor WebService Guild | OAS-61-001 | Implement `/.well-known/openapi` discovery endpoint with spec version metadata. |
| EXCITITOR-WEB-OAS-61-002 | TODO | Excititor WebService Guild | APIGOV-61-001 | Standardize error envelope responses and update controller/unit tests. |
| EXCITITOR-WEB-OAS-62-001 | TODO | Excititor WebService Guild | EXCITITOR-OAS-61-002 | Add curated examples for VEX observation/linkset endpoints and ensure portal displays them. |
| EXCITITOR-WEB-OAS-63-001 | TODO | Excititor WebService Guild, API Governance Guild | APIGOV-63-001 | Emit deprecation headers and update docs for retiring VEX APIs. |

View File

@@ -1,7 +1,9 @@
using System.Text.Json;
using Microsoft.Extensions.Options;
using StellaOps.Aoc;
using RawVexDocument = StellaOps.Concelier.RawModels.VexRawDocument;
using System.Diagnostics;
using System.Text.Json;
using Microsoft.Extensions.Options;
using StellaOps.Aoc;
using RawVexDocument = StellaOps.Concelier.RawModels.VexRawDocument;
using StellaOps.Ingestion.Telemetry;
namespace StellaOps.Excititor.Core.Aoc;
@@ -21,15 +23,35 @@ public sealed class VexRawWriteGuard : IVexRawWriteGuard
_options = options?.Value ?? AocGuardOptions.Default;
}
public void EnsureValid(RawVexDocument document)
{
ArgumentNullException.ThrowIfNull(document);
using var payload = JsonDocument.Parse(JsonSerializer.Serialize(document, SerializerOptions));
var result = _guard.Validate(payload.RootElement, _options);
if (!result.IsValid)
{
throw new ExcititorAocGuardException(result);
}
}
}
public void EnsureValid(RawVexDocument document)
{
ArgumentNullException.ThrowIfNull(document);
using var guardActivity = IngestionTelemetry.StartGuardActivity(
document.Tenant,
document.Source.Vendor,
document.Upstream.UpstreamId,
document.Upstream.ContentHash,
document.Supersedes);
using var payload = JsonDocument.Parse(JsonSerializer.Serialize(document, SerializerOptions));
var result = _guard.Validate(payload.RootElement, _options);
if (!result.IsValid)
{
var violationCount = result.Violations.IsDefaultOrEmpty ? 0 : result.Violations.Length;
var primaryCode = violationCount > 0 ? result.Violations[0].ErrorCode : string.Empty;
guardActivity?.SetTag("violationCount", violationCount);
if (!string.IsNullOrWhiteSpace(primaryCode))
{
guardActivity?.SetTag("code", primaryCode);
}
guardActivity?.SetStatus(ActivityStatusCode.Error, primaryCode);
throw new ExcititorAocGuardException(result);
}
guardActivity?.SetTag("violationCount", 0);
guardActivity?.SetStatus(ActivityStatusCode.Ok);
}
}

View File

@@ -13,5 +13,6 @@
<ItemGroup>
<ProjectReference Include="../../../Aoc/__Libraries/StellaOps.Aoc/StellaOps.Aoc.csproj" />
<ProjectReference Include="../../../Concelier/__Libraries/StellaOps.Concelier.RawModels/StellaOps.Concelier.RawModels.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.Ingestion.Telemetry/StellaOps.Ingestion.Telemetry.csproj" />
</ItemGroup>
</Project>

View File

@@ -62,7 +62,7 @@
## Observability & Forensics (Epic 15)
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-OBS-50-001 `Telemetry adoption` | TODO | Excititor Core Guild, Observability Guild | TELEMETRY-OBS-50-001, TELEMETRY-OBS-50-002 | Integrate telemetry core across VEX ingestion/linking, ensuring spans/logs capture tenant, product scope, upstream id, justification hash, and trace IDs. |
| EXCITITOR-OBS-50-001 `Telemetry adoption` | DONE (2025-11-07) | Excititor Core Guild, Observability Guild | TELEMETRY-OBS-50-001, TELEMETRY-OBS-50-002 | Integrate telemetry core across VEX ingestion/linking, ensuring spans/logs capture tenant, product scope, upstream id, justification hash, and trace IDs. |
| EXCITITOR-OBS-51-001 `Metrics & SLOs` | TODO | Excititor Core Guild, DevOps Guild | EXCITITOR-OBS-50-001, TELEMETRY-OBS-51-001 | Publish metrics for VEX ingest latency, scope resolution success, conflict rate, signature verification failures. Define SLOs (link latency P95 <30s) and configure burn-rate alerts. |
| EXCITITOR-OBS-52-001 `Timeline events` | TODO | Excititor Core Guild | EXCITITOR-OBS-50-001, TIMELINE-OBS-52-002 | Emit `timeline_event` entries for VEX ingest/linking/outcome changes with trace IDs, justification summaries, and evidence placeholders. |
| EXCITITOR-OBS-53-001 `Evidence snapshots` | TODO | Excititor Core Guild, Evidence Locker Guild | EXCITITOR-OBS-52-001, EVID-OBS-53-002 | Build evidence payloads for VEX statements (raw doc, normalization diff, precedence notes) and push to evidence locker with Merkle manifests. |

View File

@@ -68,6 +68,8 @@ public interface IVexConnectorStateRepository
ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null);
ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null);
ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null);
}
public interface IVexConsensusHoldStore

View File

@@ -1,9 +1,10 @@
using System;
using System.Collections.Immutable;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Driver;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Driver;
namespace StellaOps.Excititor.Storage.Mongo;
@@ -29,11 +30,11 @@ public sealed class MongoVexConnectorStateRepository : IVexConnectorStateReposit
return document?.ToRecord();
}
public async ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
ArgumentNullException.ThrowIfNull(state);
var document = VexConnectorStateDocument.FromRecord(state.WithNormalizedDigests());
public async ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
ArgumentNullException.ThrowIfNull(state);
var document = VexConnectorStateDocument.FromRecord(state.WithNormalizedDigests());
var filter = Builders<VexConnectorStateDocument>.Filter.Eq(x => x.ConnectorId, document.ConnectorId);
if (session is null)
{
@@ -41,10 +42,24 @@ public sealed class MongoVexConnectorStateRepository : IVexConnectorStateReposit
}
else
{
await _collection.ReplaceOneAsync(session, filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false);
}
}
}
await _collection.ReplaceOneAsync(session, filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false);
}
}
public async ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
var find = session is null
? _collection.Find(FilterDefinition<VexConnectorStateDocument>.Empty)
: _collection.Find(session, FilterDefinition<VexConnectorStateDocument>.Empty);
var documents = await find
.SortBy(x => x.ConnectorId)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return documents.ConvertAll(static document => document.ToRecord());
}
}
internal static class VexConnectorStateExtensions
{

View File

@@ -1,36 +1,34 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.ComponentModel.DataAnnotations;
using System.Diagnostics;
using System.IO;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using MongoDB.Driver;
using MongoDB.Driver.Core.Clusters;
using MongoDB.Driver.Core.Clusters;
using MongoDB.Driver.GridFS;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Aoc;
using RawContentMetadata = StellaOps.Concelier.RawModels.RawContent;
using RawLinkset = StellaOps.Concelier.RawModels.RawLinkset;
using RawReference = StellaOps.Concelier.RawModels.RawReference;
using RawSignatureMetadata = StellaOps.Concelier.RawModels.RawSignatureMetadata;
using RawSourceMetadata = StellaOps.Concelier.RawModels.RawSourceMetadata;
using RawUpstreamMetadata = StellaOps.Concelier.RawModels.RawUpstreamMetadata;
using RawVexDocument = StellaOps.Concelier.RawModels.VexRawDocument;
using VexStatementSummaryModel = StellaOps.Concelier.RawModels.VexStatementSummary;
using StellaOps.Ingestion.Telemetry;
namespace StellaOps.Excititor.Storage.Mongo;
public sealed class MongoVexRawStore : IVexRawStore
{
private readonly IMongoClient _client;
private readonly IMongoClient _client;
private readonly IMongoCollection<VexRawDocumentRecord> _collection;
private readonly GridFSBucket _bucket;
private readonly VexMongoStorageOptions _options;
private readonly IVexMongoSessionProvider _sessionProvider;
private readonly IVexRawWriteGuard _guard;
private readonly ILogger<MongoVexRawStore> _logger;
private readonly string _connectorVersion;
public MongoVexRawStore(
@@ -38,13 +36,15 @@ public sealed class MongoVexRawStore : IVexRawStore
IMongoDatabase database,
IOptions<VexMongoStorageOptions> options,
IVexMongoSessionProvider sessionProvider,
IVexRawWriteGuard guard)
IVexRawWriteGuard guard,
ILogger<MongoVexRawStore>? logger = null)
{
_client = client ?? throw new ArgumentNullException(nameof(client));
ArgumentNullException.ThrowIfNull(database);
ArgumentNullException.ThrowIfNull(options);
_sessionProvider = sessionProvider ?? throw new ArgumentNullException(nameof(sessionProvider));
_guard = guard ?? throw new ArgumentNullException(nameof(guard));
_logger = logger ?? NullLogger<MongoVexRawStore>.Instance;
_options = options.Value;
Validator.ValidateObject(_options, new ValidationContext(_options), validateAllProperties: true);
@@ -54,350 +54,285 @@ public sealed class MongoVexRawStore : IVexRawStore
_collection = database.GetCollection<VexRawDocumentRecord>(VexMongoCollectionNames.Raw);
_bucket = new GridFSBucket(database, new GridFSBucketOptions
{
BucketName = _options.RawBucketName,
ReadConcern = database.Settings.ReadConcern,
ReadPreference = database.Settings.ReadPreference,
WriteConcern = database.Settings.WriteConcern,
});
}
BucketName = _options.RawBucketName,
ReadConcern = database.Settings.ReadConcern,
ReadPreference = database.Settings.ReadPreference,
WriteConcern = database.Settings.WriteConcern,
});
}
public async ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
ArgumentNullException.ThrowIfNull(document);
var guardPayload = CreateRawModel(document);
_guard.EnsureValid(guardPayload);
var guardPayload = VexRawDocumentMapper.ToRawModel(document, _options.DefaultTenant);
var tenant = guardPayload.Tenant;
var sourceVendor = guardPayload.Source.Vendor;
var upstreamId = guardPayload.Upstream.UpstreamId;
var contentHash = guardPayload.Upstream.ContentHash;
using var logScope = _logger.BeginScope(new Dictionary<string, object?>(StringComparer.Ordinal)
{
["tenant"] = tenant,
["source.vendor"] = sourceVendor,
["upstream.upstreamId"] = upstreamId,
["contentHash"] = contentHash,
["providerId"] = document.ProviderId,
["digest"] = document.Digest,
});
var transformWatch = Stopwatch.StartNew();
using var transformActivity = IngestionTelemetry.StartTransformActivity(
tenant,
sourceVendor,
upstreamId,
contentHash,
document.Format.ToString(),
document.Content.Length);
try
{
_guard.EnsureValid(guardPayload);
transformActivity?.SetStatus(ActivityStatusCode.Ok);
}
catch (ExcititorAocGuardException ex)
{
transformActivity?.SetTag("violationCount", ex.Violations.IsDefaultOrEmpty ? 0 : ex.Violations.Length);
transformActivity?.SetTag("code", ex.PrimaryErrorCode);
transformActivity?.SetStatus(ActivityStatusCode.Error, ex.PrimaryErrorCode);
IngestionTelemetry.RecordViolation(tenant, sourceVendor, ex.PrimaryErrorCode);
IngestionTelemetry.RecordWriteAttempt(tenant, sourceVendor, IngestionTelemetry.ResultReject);
_logger.LogWarning(ex, "AOC guard rejected VEX document digest={Digest} provider={ProviderId}", document.Digest, document.ProviderId);
throw;
}
finally
{
if (transformWatch.IsRunning)
{
transformWatch.Stop();
}
IngestionTelemetry.RecordLatency(tenant, sourceVendor, IngestionTelemetry.PhaseTransform, transformWatch.Elapsed);
}
var threshold = _options.GridFsInlineThresholdBytes;
var useInline = threshold == 0 || document.Content.Length <= threshold;
string? newGridId = null;
string? oldGridIdToDelete = null;
var sessionHandle = session ?? await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
if (!useInline)
{
newGridId = await UploadToGridFsAsync(document, sessionHandle, cancellationToken).ConfigureAwait(false);
}
var supportsTransactions = sessionHandle.Client.Cluster.Description.Type != ClusterType.Standalone
&& !sessionHandle.IsInTransaction;
var startedTransaction = false;
if (supportsTransactions)
{
try
{
sessionHandle.StartTransaction();
startedTransaction = true;
}
catch (NotSupportedException)
{
supportsTransactions = false;
}
}
try
{
var filter = Builders<VexRawDocumentRecord>.Filter.Eq(x => x.Id, document.Digest);
var existing = await _collection
.Find(sessionHandle, filter)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
var record = VexRawDocumentRecord.FromDomain(document, includeContent: useInline);
record.GridFsObjectId = useInline ? null : newGridId;
await _collection
.ReplaceOneAsync(
sessionHandle,
filter,
record,
new ReplaceOptions { IsUpsert = true },
cancellationToken)
.ConfigureAwait(false);
if (existing?.GridFsObjectId is string oldGridId && !string.IsNullOrWhiteSpace(oldGridId))
{
if (useInline || !string.Equals(newGridId, oldGridId, StringComparison.Ordinal))
{
oldGridIdToDelete = oldGridId;
}
}
if (startedTransaction)
{
await sessionHandle.CommitTransactionAsync(cancellationToken).ConfigureAwait(false);
}
}
catch
{
if (startedTransaction && sessionHandle.IsInTransaction)
{
await sessionHandle.AbortTransactionAsync(cancellationToken).ConfigureAwait(false);
}
if (!useInline && !string.IsNullOrWhiteSpace(newGridId))
{
await DeleteFromGridFsAsync(newGridId, sessionHandle, cancellationToken).ConfigureAwait(false);
}
throw;
}
if (!string.IsNullOrWhiteSpace(oldGridIdToDelete))
{
await DeleteFromGridFsAsync(oldGridIdToDelete!, sessionHandle, cancellationToken).ConfigureAwait(false);
}
}
public async ValueTask<VexRawDocument?> FindByDigestAsync(string digest, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
if (string.IsNullOrWhiteSpace(digest))
{
throw new ArgumentException("Digest must be provided.", nameof(digest));
}
var trimmed = digest.Trim();
var filter = Builders<VexRawDocumentRecord>.Filter.Eq(x => x.Id, trimmed);
var record = session is null
? await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false)
: await _collection.Find(session, filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
if (record is null)
{
return null;
}
if (!string.IsNullOrWhiteSpace(record.GridFsObjectId))
{
var handle = session ?? await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
var bytes = await DownloadFromGridFsAsync(record.GridFsObjectId, handle, cancellationToken).ConfigureAwait(false);
return record.ToDomain(new ReadOnlyMemory<byte>(bytes));
}
return record.ToDomain();
}
private async Task<string?> UploadToGridFsAsync(VexRawDocument document, IClientSessionHandle? session, CancellationToken cancellationToken)
{
using var stream = new MemoryStream(document.Content.ToArray(), writable: false);
var metadata = new BsonDocument
{
{ "providerId", document.ProviderId },
{ "format", document.Format.ToString().ToLowerInvariant() },
{ "sourceUri", document.SourceUri.ToString() },
{ "retrievedAt", document.RetrievedAt.UtcDateTime },
};
var options = new GridFSUploadOptions { Metadata = metadata };
var objectId = await _bucket
.UploadFromStreamAsync(document.Digest, stream, options, cancellationToken)
.ConfigureAwait(false);
return objectId.ToString();
}
private async Task DeleteFromGridFsAsync(string gridFsObjectId, IClientSessionHandle? session, CancellationToken cancellationToken)
{
if (!ObjectId.TryParse(gridFsObjectId, out var objectId))
{
return;
}
try
{
await _bucket.DeleteAsync(objectId, cancellationToken).ConfigureAwait(false);
}
catch (GridFSFileNotFoundException)
{
// file already removed by TTL or manual cleanup
}
}
private async Task<byte[]> DownloadFromGridFsAsync(string gridFsObjectId, IClientSessionHandle? session, CancellationToken cancellationToken)
{
if (!ObjectId.TryParse(gridFsObjectId, out var objectId))
{
return Array.Empty<byte>();
}
return await _bucket.DownloadAsBytesAsync(objectId, null, cancellationToken).ConfigureAwait(false);
}
async ValueTask IVexRawDocumentSink.StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
=> await StoreAsync(document, cancellationToken, session: null).ConfigureAwait(false);
var sessionHandle = session ?? await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
private RawVexDocument CreateRawModel(VexRawDocument document)
{
var metadata = document.Metadata ?? ImmutableDictionary<string, string>.Empty;
var tenant = _options.DefaultTenant;
if (!useInline)
{
newGridId = await UploadToGridFsAsync(document, sessionHandle, cancellationToken).ConfigureAwait(false);
}
var source = CreateSourceMetadata(document, metadata);
var content = CreateContent(document, metadata);
var upstream = CreateUpstreamMetadata(document, metadata);
var linkset = CreateLinkset();
var statements = ImmutableArray<VexStatementSummaryModel>.Empty;
var supportsTransactions = sessionHandle.Client.Cluster.Description.Type != ClusterType.Standalone
&& !sessionHandle.IsInTransaction;
return new RawVexDocument(
var startedTransaction = false;
if (supportsTransactions)
{
try
{
sessionHandle.StartTransaction();
startedTransaction = true;
}
catch (NotSupportedException)
{
supportsTransactions = false;
}
}
var fetchWatch = Stopwatch.StartNew();
using var fetchActivity = IngestionTelemetry.StartFetchActivity(
tenant,
source,
upstream,
content,
linkset,
statements);
}
private RawSourceMetadata CreateSourceMetadata(VexRawDocument document, ImmutableDictionary<string, string> metadata)
{
var vendor = TryMetadata(metadata, "source.vendor", "connector.vendor") ?? ExtractVendor(document.ProviderId);
var connector = TryMetadata(metadata, "source.connector") ?? document.ProviderId;
var version = TryMetadata(metadata, "source.connector_version", "connector.version") ?? _connectorVersion;
var stream = TryMetadata(metadata, "source.stream", "connector.stream") ?? document.Format.ToString().ToLowerInvariant();
return new RawSourceMetadata(
vendor,
connector,
version,
stream);
}
private RawUpstreamMetadata CreateUpstreamMetadata(VexRawDocument document, ImmutableDictionary<string, string> metadata)
{
var upstreamId = TryMetadata(
metadata,
"upstream.id",
"aoc.upstream_id",
"vulnerability.id",
"advisory.id",
"msrc.vulnerabilityId",
"msrc.advisoryId",
"oracle.csaf.entryId",
"ubuntu.advisoryId",
"cisco.csaf.documentId",
"rancher.vex.id") ?? document.SourceUri.ToString();
var documentVersion = TryMetadata(
metadata,
"upstream.version",
"aoc.document_version",
"msrc.lastModified",
"msrc.releaseDate",
"oracle.csaf.revision",
"ubuntu.version",
"ubuntu.lastModified",
"cisco.csaf.revision") ?? document.RetrievedAt.ToString("O");
var signature = CreateSignatureMetadata(metadata);
var provenance = metadata;
return new RawUpstreamMetadata(
sourceVendor,
upstreamId,
documentVersion,
document.RetrievedAt,
document.Digest,
signature,
provenance);
}
contentHash,
document.SourceUri.ToString());
fetchActivity?.SetTag("providerId", document.ProviderId);
fetchActivity?.SetTag("format", document.Format.ToString().ToLowerInvariant());
private static RawSignatureMetadata CreateSignatureMetadata(ImmutableDictionary<string, string> metadata)
{
if (!TryBool(metadata, out var present, "signature.present", "aoc.signature.present"))
VexRawDocumentRecord? existing;
try
{
return new RawSignatureMetadata(false);
var filter = Builders<VexRawDocumentRecord>.Filter.Eq(x => x.Id, document.Digest);
existing = await _collection
.Find(sessionHandle, filter)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
fetchActivity?.SetTag("result", existing is null ? "miss" : "hit");
fetchActivity?.SetStatus(ActivityStatusCode.Ok);
}
catch
{
fetchActivity?.SetStatus(ActivityStatusCode.Error, "lookup-failed");
throw;
}
finally
{
if (fetchWatch.IsRunning)
{
fetchWatch.Stop();
}
IngestionTelemetry.RecordLatency(tenant, sourceVendor, IngestionTelemetry.PhaseFetch, fetchWatch.Elapsed);
}
if (!present)
var record = VexRawDocumentRecord.FromDomain(document, includeContent: useInline);
record.GridFsObjectId = useInline ? null : newGridId;
var writeWatch = Stopwatch.StartNew();
using var writeActivity = IngestionTelemetry.StartWriteActivity(
tenant,
sourceVendor,
upstreamId,
contentHash,
VexMongoCollectionNames.Raw);
string? writeResult = null;
try
{
return new RawSignatureMetadata(false);
await _collection
.ReplaceOneAsync(
sessionHandle,
Builders<VexRawDocumentRecord>.Filter.Eq(x => x.Id, document.Digest),
record,
new ReplaceOptions { IsUpsert = true },
cancellationToken)
.ConfigureAwait(false);
writeResult = existing is null ? IngestionTelemetry.ResultOk : IngestionTelemetry.ResultNoop;
writeActivity?.SetTag("result", writeResult);
if (existing?.GridFsObjectId is string oldGridId && !string.IsNullOrWhiteSpace(oldGridId))
{
if (useInline || !string.Equals(newGridId, oldGridId, StringComparison.Ordinal))
{
oldGridIdToDelete = oldGridId;
}
}
if (startedTransaction)
{
await sessionHandle.CommitTransactionAsync(cancellationToken).ConfigureAwait(false);
}
writeActivity?.SetStatus(ActivityStatusCode.Ok);
}
catch
{
if (startedTransaction && sessionHandle.IsInTransaction)
{
await sessionHandle.AbortTransactionAsync(cancellationToken).ConfigureAwait(false);
}
if (!useInline && !string.IsNullOrWhiteSpace(newGridId))
{
await DeleteFromGridFsAsync(newGridId, sessionHandle, cancellationToken).ConfigureAwait(false);
}
writeActivity?.SetStatus(ActivityStatusCode.Error, "write-failed");
throw;
}
finally
{
if (writeWatch.IsRunning)
{
writeWatch.Stop();
}
IngestionTelemetry.RecordLatency(tenant, sourceVendor, IngestionTelemetry.PhaseWrite, writeWatch.Elapsed);
if (!string.IsNullOrEmpty(writeResult))
{
IngestionTelemetry.RecordWriteAttempt(tenant, sourceVendor, writeResult);
}
}
var format = TryMetadata(metadata, "signature.format", "aoc.signature.format");
var keyId = TryMetadata(metadata, "signature.key_id", "signature.keyId", "aoc.signature.key_id");
var signature = TryMetadata(metadata, "signature.sig", "signature.signature", "aoc.signature.sig");
var digest = TryMetadata(metadata, "signature.digest", "aoc.signature.digest");
var certificate = TryMetadata(metadata, "signature.certificate", "aoc.signature.certificate");
return new RawSignatureMetadata(true, format, keyId, signature, certificate, digest);
if (!string.IsNullOrWhiteSpace(oldGridIdToDelete))
{
await DeleteFromGridFsAsync(oldGridIdToDelete!, sessionHandle, cancellationToken).ConfigureAwait(false);
}
}
private RawContentMetadata CreateContent(VexRawDocument document, ImmutableDictionary<string, string> metadata)
public async ValueTask<VexRawDocument?> FindByDigestAsync(string digest, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
if (document.Content.IsEmpty)
if (string.IsNullOrWhiteSpace(digest))
{
throw new InvalidOperationException("Raw VEX document content cannot be empty when enforcing AOC guard.");
throw new ArgumentException("Digest must be provided.", nameof(digest));
}
var trimmed = digest.Trim();
var filter = Builders<VexRawDocumentRecord>.Filter.Eq(x => x.Id, trimmed);
var record = session is null
? await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false)
: await _collection.Find(session, filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
if (record is null)
{
return null;
}
if (!string.IsNullOrWhiteSpace(record.GridFsObjectId))
{
var handle = session ?? await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
var bytes = await DownloadFromGridFsAsync(record.GridFsObjectId, handle, cancellationToken).ConfigureAwait(false);
return record.ToDomain(new ReadOnlyMemory<byte>(bytes));
}
return record.ToDomain();
}
private async Task<string?> UploadToGridFsAsync(VexRawDocument document, IClientSessionHandle? session, CancellationToken cancellationToken)
{
using var stream = new MemoryStream(document.Content.ToArray(), writable: false);
var metadata = new BsonDocument
{
{ "providerId", document.ProviderId },
{ "format", document.Format.ToString().ToLowerInvariant() },
{ "sourceUri", document.SourceUri.ToString() },
{ "retrievedAt", document.RetrievedAt.UtcDateTime },
};
var options = new GridFSUploadOptions { Metadata = metadata };
var objectId = await _bucket
.UploadFromStreamAsync(document.Digest, stream, options, cancellationToken)
.ConfigureAwait(false);
return objectId.ToString();
}
private async Task DeleteFromGridFsAsync(string gridFsObjectId, IClientSessionHandle? session, CancellationToken cancellationToken)
{
if (!ObjectId.TryParse(gridFsObjectId, out var objectId))
{
return;
}
try
{
using var payload = JsonDocument.Parse(document.Content.ToArray());
var raw = payload.RootElement.Clone();
var specVersion = TryMetadata(metadata, "content.spec_version", "csaf.version", "openvex.version");
var encoding = TryMetadata(metadata, "content.encoding");
return new RawContentMetadata(
document.Format.ToString(),
specVersion,
raw,
encoding);
await _bucket.DeleteAsync(objectId, cancellationToken).ConfigureAwait(false);
}
catch (JsonException ex)
catch (GridFSFileNotFoundException)
{
throw new InvalidOperationException("Raw VEX document payload must be valid JSON for AOC guard enforcement.", ex);
// file already removed by TTL or manual cleanup
}
}
private static RawLinkset CreateLinkset()
=> new()
{
Aliases = ImmutableArray<string>.Empty,
PackageUrls = ImmutableArray<string>.Empty,
Cpes = ImmutableArray<string>.Empty,
References = ImmutableArray<RawReference>.Empty,
ReconciledFrom = ImmutableArray<string>.Empty,
Notes = ImmutableDictionary<string, string>.Empty,
};
private static string? TryMetadata(ImmutableDictionary<string, string> metadata, params string[] keys)
private async Task<byte[]> DownloadFromGridFsAsync(string gridFsObjectId, IClientSessionHandle? session, CancellationToken cancellationToken)
{
foreach (var key in keys)
if (!ObjectId.TryParse(gridFsObjectId, out var objectId))
{
if (metadata.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
{
return value;
}
return Array.Empty<byte>();
}
return null;
return await _bucket.DownloadAsBytesAsync(objectId, null, cancellationToken).ConfigureAwait(false);
}
private static bool TryBool(ImmutableDictionary<string, string> metadata, out bool value, params string[] keys)
{
foreach (var key in keys)
{
if (metadata.TryGetValue(key, out var text) && bool.TryParse(text, out value))
{
return true;
}
}
value = default;
return false;
}
private static string ExtractVendor(string providerId)
{
if (string.IsNullOrWhiteSpace(providerId))
{
return "unknown";
}
var trimmed = providerId.Trim();
var separatorIndex = trimmed.LastIndexOfAny(new[] { ':', '.' });
if (separatorIndex >= 0 && separatorIndex < trimmed.Length - 1)
{
return trimmed[(separatorIndex + 1)..];
}
return trimmed;
}
async ValueTask IVexRawDocumentSink.StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
=> await StoreAsync(document, cancellationToken, session: null).ConfigureAwait(false);
}

View File

@@ -12,7 +12,8 @@
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
</ItemGroup>
</Project>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
<ProjectReference Include="..\..\..\__Libraries\StellaOps.Ingestion.Telemetry\StellaOps.Ingestion.Telemetry.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,194 @@
using System;
using System.Collections.Immutable;
using System.Text.Json;
using StellaOps.Excititor.Core;
using RawVexDocumentModel = StellaOps.Concelier.RawModels.VexRawDocument;
using RawSourceMetadata = StellaOps.Concelier.RawModels.RawSourceMetadata;
using RawUpstreamMetadata = StellaOps.Concelier.RawModels.RawUpstreamMetadata;
using RawContentMetadata = StellaOps.Concelier.RawModels.RawContent;
using RawSignatureMetadata = StellaOps.Concelier.RawModels.RawSignatureMetadata;
using RawLinkset = StellaOps.Concelier.RawModels.RawLinkset;
using RawReference = StellaOps.Concelier.RawModels.RawReference;
using VexStatementSummaryModel = StellaOps.Concelier.RawModels.VexStatementSummary;
namespace StellaOps.Excititor.Storage.Mongo;
/// <summary>
/// Converts Excititor domain VEX documents into Aggregation-Only Contract raw payloads.
/// </summary>
public static class VexRawDocumentMapper
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
public static RawVexDocumentModel ToRawModel(VexRawDocument document, string defaultTenant)
{
ArgumentNullException.ThrowIfNull(document);
var metadata = document.Metadata ?? ImmutableDictionary<string, string>.Empty;
var tenant = ResolveTenant(metadata, defaultTenant);
var source = CreateSourceMetadata(document, metadata);
var upstream = CreateUpstreamMetadata(document, metadata);
var content = CreateContent(document, metadata);
var linkset = CreateLinkset();
ImmutableArray<VexStatementSummaryModel>? statements = null;
return new RawVexDocumentModel(tenant, source, upstream, content, linkset, statements);
}
private static string ResolveTenant(ImmutableDictionary<string, string> metadata, string fallback)
{
var tenant = TryMetadata(metadata, "tenant", "aoc.tenant");
if (string.IsNullOrWhiteSpace(tenant))
{
return (fallback ?? "tenant-default").Trim().ToLowerInvariant();
}
return tenant.Trim().ToLowerInvariant();
}
private static RawSourceMetadata CreateSourceMetadata(VexRawDocument document, ImmutableDictionary<string, string> metadata)
{
var vendor = TryMetadata(metadata, "source.vendor", "connector.vendor") ?? ExtractVendor(document.ProviderId);
var connector = TryMetadata(metadata, "source.connector") ?? document.ProviderId;
var version = TryMetadata(metadata, "source.connector_version", "connector.version") ?? GetAssemblyVersion();
var stream = TryMetadata(metadata, "source.stream", "connector.stream") ?? document.Format.ToString().ToLowerInvariant();
return new RawSourceMetadata(vendor, connector, version, stream);
}
private static string GetAssemblyVersion()
=> typeof(VexRawDocumentMapper).Assembly.GetName().Version?.ToString() ?? "0.0.0";
private static RawUpstreamMetadata CreateUpstreamMetadata(VexRawDocument document, ImmutableDictionary<string, string> metadata)
{
var upstreamId = TryMetadata(
metadata,
"upstream.id",
"aoc.upstream_id",
"vulnerability.id",
"advisory.id",
"msrc.vulnerabilityId",
"msrc.advisoryId",
"oracle.csaf.entryId",
"ubuntu.advisoryId",
"cisco.csaf.documentId",
"rancher.vex.id") ?? document.SourceUri.ToString();
var documentVersion = TryMetadata(
metadata,
"upstream.version",
"aoc.document_version",
"msrc.lastModified",
"msrc.releaseDate",
"oracle.csaf.revision",
"ubuntu.version",
"ubuntu.lastModified",
"cisco.csaf.revision") ?? document.RetrievedAt.ToString("O");
var signature = CreateSignatureMetadata(metadata);
return new RawUpstreamMetadata(
upstreamId,
documentVersion,
document.RetrievedAt,
document.Digest,
signature,
metadata);
}
private static RawSignatureMetadata CreateSignatureMetadata(ImmutableDictionary<string, string> metadata)
{
if (!TryBool(metadata, out var present, "signature.present", "aoc.signature.present"))
{
return new RawSignatureMetadata(false);
}
if (!present)
{
return new RawSignatureMetadata(false);
}
var format = TryMetadata(metadata, "signature.format", "aoc.signature.format");
var keyId = TryMetadata(metadata, "signature.key_id", "signature.keyId", "aoc.signature.key_id");
var signature = TryMetadata(metadata, "signature.sig", "signature.signature", "aoc.signature.sig");
var digest = TryMetadata(metadata, "signature.digest", "aoc.signature.digest");
var certificate = TryMetadata(metadata, "signature.certificate", "aoc.signature.certificate");
return new RawSignatureMetadata(true, format, keyId, signature, certificate, digest);
}
private static RawContentMetadata CreateContent(VexRawDocument document, ImmutableDictionary<string, string> metadata)
{
if (document.Content.IsEmpty)
{
throw new InvalidOperationException("Raw VEX document content cannot be empty when enforcing AOC guard.");
}
try
{
using var payload = JsonDocument.Parse(document.Content.ToArray());
var raw = payload.RootElement.Clone();
var specVersion = TryMetadata(metadata, "content.spec_version", "csaf.version", "openvex.version");
var encoding = TryMetadata(metadata, "content.encoding");
return new RawContentMetadata(
document.Format.ToString(),
specVersion,
raw,
encoding);
}
catch (JsonException ex)
{
throw new InvalidOperationException("Raw VEX document payload must be valid JSON for AOC guard enforcement.", ex);
}
}
private static RawLinkset CreateLinkset()
=> new()
{
Aliases = ImmutableArray<string>.Empty,
PackageUrls = ImmutableArray<string>.Empty,
Cpes = ImmutableArray<string>.Empty,
References = ImmutableArray<RawReference>.Empty,
ReconciledFrom = ImmutableArray<string>.Empty,
Notes = ImmutableDictionary<string, string>.Empty,
};
private static string? TryMetadata(ImmutableDictionary<string, string> metadata, params string[] keys)
{
foreach (var key in keys)
{
if (metadata.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
{
return value;
}
}
return null;
}
private static bool TryBool(ImmutableDictionary<string, string> metadata, out bool value, params string[] keys)
{
foreach (var key in keys)
{
if (metadata.TryGetValue(key, out var text) && bool.TryParse(text, out value))
{
return true;
}
}
value = default;
return false;
}
private static string ExtractVendor(string providerId)
{
if (string.IsNullOrWhiteSpace(providerId))
{
return "unknown";
}
var trimmed = providerId.Trim();
var separatorIndex = trimmed.LastIndexOfAny(new[] { ':', '.' });
if (separatorIndex >= 0 && separatorIndex < trimmed.Length - 1)
{
return trimmed[(separatorIndex + 1)..];
}
return trimmed;
}
}

View File

@@ -1,23 +1,20 @@
using System.Collections.Generic;
using Microsoft.Extensions.Logging.Abstractions;
using Mongo2Go;
using MongoDB.Bson;
using MongoDB.Driver;
namespace StellaOps.Excititor.Storage.Mongo.Tests;
public sealed class MongoVexCacheMaintenanceTests : IAsyncLifetime
{
private readonly MongoDbRunner _runner;
private readonly IMongoDatabase _database;
public MongoVexCacheMaintenanceTests()
{
_runner = MongoDbRunner.Start();
var client = new MongoClient(_runner.ConnectionString);
_database = client.GetDatabase("vex-cache-maintenance-tests");
VexMongoMappingRegistry.Register();
}
using Microsoft.Extensions.Logging.Abstractions;
using MongoDB.Bson;
using MongoDB.Driver;
namespace StellaOps.Excititor.Storage.Mongo.Tests;
public sealed class MongoVexCacheMaintenanceTests : IAsyncLifetime
{
private readonly TestMongoEnvironment _mongo = new();
private readonly IMongoDatabase _database;
public MongoVexCacheMaintenanceTests()
{
_database = _mongo.CreateDatabase("cache-maintenance");
VexMongoMappingRegistry.Register();
}
[Fact]
public async Task RemoveExpiredAsync_DeletesEntriesBeforeCutoff()
@@ -114,9 +111,5 @@ public sealed class MongoVexCacheMaintenanceTests : IAsyncLifetime
public Task InitializeAsync() => Task.CompletedTask;
public Task DisposeAsync()
{
_runner.Dispose();
return Task.CompletedTask;
}
}
public Task DisposeAsync() => _mongo.DisposeAsync();
}

View File

@@ -3,7 +3,6 @@ using System.Globalization;
using System.Linq;
using System.Text;
using Microsoft.Extensions.Options;
using Mongo2Go;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Aoc;
@@ -13,21 +12,20 @@ using RawVexDocumentModel = StellaOps.Concelier.RawModels.VexRawDocument;
namespace StellaOps.Excititor.Storage.Mongo.Tests;
public sealed class MongoVexRepositoryTests : IAsyncLifetime
{
private readonly MongoDbRunner _runner;
private readonly MongoClient _client;
public MongoVexRepositoryTests()
{
_runner = MongoDbRunner.Start();
_client = new MongoClient(_runner.ConnectionString);
}
public sealed class MongoVexRepositoryTests : IAsyncLifetime
{
private readonly TestMongoEnvironment _mongo = new();
private readonly MongoClient _client;
public MongoVexRepositoryTests()
{
_client = _mongo.Client;
}
[Fact]
public async Task RawStore_UsesGridFsForLargePayloads()
{
var database = _client.GetDatabase($"vex-raw-gridfs-{Guid.NewGuid():N}");
var database = _mongo.CreateDatabase("vex-raw-gridfs");
var store = CreateRawStore(database, thresholdBytes: 32);
var payload = CreateJsonPayload(new string('A', 256));
@@ -63,7 +61,7 @@ public sealed class MongoVexRepositoryTests : IAsyncLifetime
[Fact]
public async Task RawStore_ReplacesGridFsWithInlinePayload()
{
var database = _client.GetDatabase($"vex-raw-inline-{Guid.NewGuid():N}");
var database = _mongo.CreateDatabase("vex-raw-inline");
var store = CreateRawStore(database, thresholdBytes: 16);
var largePayload = CreateJsonPayload(new string('B', 128));
@@ -176,7 +174,7 @@ public sealed class MongoVexRepositoryTests : IAsyncLifetime
[Fact]
public async Task ExportStore_FindAsync_ExpiresCacheEntries()
{
var database = _client.GetDatabase($"vex-export-expire-{Guid.NewGuid():N}");
var database = _mongo.CreateDatabase("vex-export-expire");
var options = Options.Create(new VexMongoStorageOptions
{
ExportCacheTtl = TimeSpan.FromMinutes(5),
@@ -217,7 +215,7 @@ public sealed class MongoVexRepositoryTests : IAsyncLifetime
[Fact]
public async Task ClaimStore_AppendsAndQueriesStatements()
{
var database = _client.GetDatabase($"vex-claims-{Guid.NewGuid():N}");
var database = _mongo.CreateDatabase("vex-claims");
var store = new MongoVexClaimStore(database);
var product = new VexProduct("pkg:demo/app", "Demo App", version: "1.0.0", purl: "pkg:demo/app@1.0.0");
@@ -305,11 +303,7 @@ public sealed class MongoVexRepositoryTests : IAsyncLifetime
public Task InitializeAsync() => Task.CompletedTask;
public Task DisposeAsync()
{
_runner.Dispose();
return Task.CompletedTask;
}
public Task DisposeAsync() => _mongo.DisposeAsync();
private static byte[] CreateJsonPayload(string value)
=> Encoding.UTF8.GetBytes(CreateJsonPayloadString(value));

View File

@@ -2,23 +2,23 @@ using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Mongo2Go;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Storage.Mongo.Tests;
public sealed class MongoVexSessionConsistencyTests : IAsyncLifetime
{
private readonly MongoDbRunner _runner;
public MongoVexSessionConsistencyTests()
{
_runner = MongoDbRunner.Start(singleNodeReplSet: true);
}
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Storage.Mongo.Tests;
public sealed class MongoVexSessionConsistencyTests : IAsyncLifetime
{
private readonly TestMongoEnvironment _mongo = new();
private readonly MongoClient _client;
public MongoVexSessionConsistencyTests()
{
_client = _mongo.Client;
}
[Fact]
public async Task SessionProvidesReadYourWrites()
@@ -45,7 +45,7 @@ public sealed class MongoVexSessionConsistencyTests : IAsyncLifetime
await using var provider = BuildServiceProvider();
await using var scope = provider.CreateAsyncScope();
var client = scope.ServiceProvider.GetRequiredService<IMongoClient>();
var client = scope.ServiceProvider.GetRequiredService<IMongoClient>();
var sessionProvider = scope.ServiceProvider.GetRequiredService<IVexMongoSessionProvider>();
var providerStore = scope.ServiceProvider.GetRequiredService<IVexProviderStore>();
@@ -74,18 +74,18 @@ public sealed class MongoVexSessionConsistencyTests : IAsyncLifetime
private ServiceProvider BuildServiceProvider()
{
var services = new ServiceCollection();
services.AddLogging(builder => builder.AddDebug());
services.Configure<VexMongoStorageOptions>(options =>
{
options.ConnectionString = _runner.ConnectionString;
options.DatabaseName = $"excititor-session-tests-{Guid.NewGuid():N}";
options.CommandTimeout = TimeSpan.FromSeconds(5);
options.RawBucketName = "vex.raw";
});
services.AddExcititorMongoStorage();
return services.BuildServiceProvider();
}
var services = new ServiceCollection();
services.AddLogging(builder => builder.AddDebug());
services.Configure<VexMongoStorageOptions>(options =>
{
options.ConnectionString = _mongo.ConnectionString;
options.DatabaseName = _mongo.ReserveDatabase("session");
options.CommandTimeout = TimeSpan.FromSeconds(5);
options.RawBucketName = "vex.raw";
});
services.AddExcititorMongoStorage();
return services.BuildServiceProvider();
}
private static async Task ExecuteWithRetryAsync(Func<Task> action, CancellationToken cancellationToken)
{
@@ -176,9 +176,5 @@ public sealed class MongoVexSessionConsistencyTests : IAsyncLifetime
public Task InitializeAsync() => Task.CompletedTask;
public Task DisposeAsync()
{
_runner.Dispose();
return Task.CompletedTask;
}
}
public Task DisposeAsync() => _mongo.DisposeAsync();
}

View File

@@ -3,22 +3,22 @@ using System.Collections.Immutable;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Mongo2Go;
using Microsoft.Extensions.Logging;
using System.Text;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Aoc;
namespace StellaOps.Excititor.Storage.Mongo.Tests;
public sealed class MongoVexStatementBackfillServiceTests : IAsyncLifetime
{
private readonly MongoDbRunner _runner;
public MongoVexStatementBackfillServiceTests()
{
_runner = MongoDbRunner.Start(singleNodeReplSet: true);
}
using RawVexDocumentModel = StellaOps.Concelier.RawModels.VexRawDocument;
namespace StellaOps.Excititor.Storage.Mongo.Tests;
public sealed class MongoVexStatementBackfillServiceTests : IAsyncLifetime
{
private readonly TestMongoEnvironment _mongo = new();
public MongoVexStatementBackfillServiceTests()
{
// Intentionally left blank; Mongo environment is initialized on demand.
}
[Fact]
public async Task RunAsync_BackfillsStatementsFromRawDocuments()
@@ -108,34 +108,32 @@ public sealed class MongoVexStatementBackfillServiceTests : IAsyncLifetime
var services = new ServiceCollection();
services.AddLogging(builder => builder.AddDebug());
services.AddSingleton(TimeProvider.System);
services.Configure<VexMongoStorageOptions>(options =>
{
options.ConnectionString = _runner.ConnectionString;
options.DatabaseName = $"excititor-backfill-tests-{Guid.NewGuid():N}";
options.CommandTimeout = TimeSpan.FromSeconds(5);
options.RawBucketName = "vex.raw";
options.GridFsInlineThresholdBytes = 1024;
options.ExportCacheTtl = TimeSpan.FromHours(1);
});
services.Configure<VexMongoStorageOptions>(options =>
{
options.ConnectionString = _mongo.ConnectionString;
options.DatabaseName = _mongo.ReserveDatabase("backfill");
options.CommandTimeout = TimeSpan.FromSeconds(5);
options.RawBucketName = "vex.raw";
options.GridFsInlineThresholdBytes = 1024;
options.ExportCacheTtl = TimeSpan.FromHours(1);
options.DefaultTenant = "tests";
});
services.AddExcititorMongoStorage();
services.AddExcititorAocGuards();
services.AddSingleton<IVexRawWriteGuard, PermissiveVexRawWriteGuard>();
services.AddSingleton<IVexNormalizer, TestNormalizer>();
return services.BuildServiceProvider();
}
public Task InitializeAsync() => Task.CompletedTask;
public Task DisposeAsync()
{
_runner.Dispose();
return Task.CompletedTask;
}
public Task DisposeAsync() => _mongo.DisposeAsync();
private static ReadOnlyMemory<byte> CreateJsonPayload(string value)
=> Encoding.UTF8.GetBytes($"{{\"data\":\"{value}\"}}");
private sealed class TestNormalizer : IVexNormalizer
{
private sealed class TestNormalizer : IVexNormalizer
{
public string Format => "csaf";
public bool CanHandle(VexRawDocument document) => true;
@@ -171,6 +169,14 @@ public sealed class MongoVexStatementBackfillServiceTests : IAsyncLifetime
var claims = ImmutableArray.Create(claim);
return ValueTask.FromResult(new VexClaimBatch(document, claims, ImmutableDictionary<string, string>.Empty));
}
}
}
}
}
private sealed class PermissiveVexRawWriteGuard : IVexRawWriteGuard
{
public void EnsureValid(RawVexDocumentModel document)
{
// Tests control the payloads; guard bypass keeps focus on backfill logic.
}
}
}

View File

@@ -1,23 +1,20 @@
using System.Globalization;
using Mongo2Go;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Storage.Mongo.Tests;
public sealed class MongoVexStoreMappingTests : IAsyncLifetime
{
private readonly MongoDbRunner _runner;
private readonly IMongoDatabase _database;
public MongoVexStoreMappingTests()
{
_runner = MongoDbRunner.Start();
var client = new MongoClient(_runner.ConnectionString);
_database = client.GetDatabase("excititor-storage-mapping-tests");
VexMongoMappingRegistry.Register();
}
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Storage.Mongo.Tests;
public sealed class MongoVexStoreMappingTests : IAsyncLifetime
{
private readonly TestMongoEnvironment _mongo = new();
private readonly IMongoDatabase _database;
public MongoVexStoreMappingTests()
{
_database = _mongo.CreateDatabase("storage-mapping");
VexMongoMappingRegistry.Register();
}
[Fact]
public async Task ProviderStore_RoundTrips_WithExtraFields()
@@ -259,9 +256,5 @@ public sealed class MongoVexStoreMappingTests : IAsyncLifetime
public Task InitializeAsync() => Task.CompletedTask;
public Task DisposeAsync()
{
_runner.Dispose();
return Task.CompletedTask;
}
}
public Task DisposeAsync() => _mongo.DisposeAsync();
}

View File

@@ -0,0 +1,88 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
using Mongo2Go;
using MongoDB.Bson;
using MongoDB.Driver;
namespace StellaOps.Excititor.Storage.Mongo.Tests;
internal sealed class TestMongoEnvironment : IAsyncLifetime
{
private const string Prefix = "exstor";
private readonly MongoDbRunner? _runner;
private readonly HashSet<string> _reservedDatabases = new(StringComparer.Ordinal);
public TestMongoEnvironment()
{
var overrideConnection = Environment.GetEnvironmentVariable("EXCITITOR_TEST_MONGO_URI");
if (!string.IsNullOrWhiteSpace(overrideConnection))
{
ConnectionString = overrideConnection.Trim();
Client = new MongoClient(ConnectionString);
return;
}
_runner = MongoDbRunner.Start(singleNodeReplSet: true);
ConnectionString = _runner.ConnectionString;
Client = new MongoClient(ConnectionString);
}
public MongoClient Client { get; }
public string ConnectionString { get; }
public string ReserveDatabase(string hint)
{
var baseName = string.IsNullOrWhiteSpace(hint) ? "db" : hint.ToLowerInvariant();
var builder = new StringBuilder(baseName.Length);
foreach (var ch in baseName)
{
builder.Append(char.IsLetterOrDigit(ch) ? ch : '_');
}
var slug = builder.Length == 0 ? "db" : builder.ToString();
var suffix = ObjectId.GenerateNewId().ToString();
var maxSlugLength = Math.Max(1, 60 - Prefix.Length - suffix.Length - 2);
if (slug.Length > maxSlugLength)
{
slug = slug[..maxSlugLength];
}
var name = $"{Prefix}_{slug}_{suffix}";
_reservedDatabases.Add(name);
return name;
}
public IMongoDatabase CreateDatabase(string hint)
{
var name = ReserveDatabase(hint);
return Client.GetDatabase(name);
}
public Task InitializeAsync() => Task.CompletedTask;
public async Task DisposeAsync()
{
if (_runner is not null)
{
_runner.Dispose();
return;
}
foreach (var db in _reservedDatabases)
{
try
{
await Client.DropDatabaseAsync(db);
}
catch (MongoException)
{
// best-effort cleanup when sharing a developer-managed instance.
}
}
_reservedDatabases.Clear();
}
}

View File

@@ -1,25 +1,22 @@
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
using Mongo2Go;
using MongoDB.Driver;
using StellaOps.Excititor.Storage.Mongo.Migrations;
using StellaOps.Excititor.Storage.Mongo;
using Microsoft.Extensions.Logging.Abstractions;
using MongoDB.Driver;
using StellaOps.Excititor.Storage.Mongo.Migrations;
using StellaOps.Excititor.Storage.Mongo;
namespace StellaOps.Excititor.Storage.Mongo.Tests;
public sealed class VexMongoMigrationRunnerTests : IAsyncLifetime
{
private readonly MongoDbRunner _runner;
private readonly IMongoDatabase _database;
public VexMongoMigrationRunnerTests()
{
_runner = MongoDbRunner.Start();
var client = new MongoClient(_runner.ConnectionString);
_database = client.GetDatabase("excititor-migrations-tests");
}
public sealed class VexMongoMigrationRunnerTests : IAsyncLifetime
{
private readonly TestMongoEnvironment _mongo = new();
private readonly IMongoDatabase _database;
public VexMongoMigrationRunnerTests()
{
_database = _mongo.CreateDatabase("migrations");
}
[Fact]
public async Task RunAsync_AppliesInitialIndexesOnce()
@@ -60,9 +57,5 @@ public sealed class VexMongoMigrationRunnerTests : IAsyncLifetime
public Task InitializeAsync() => Task.CompletedTask;
public Task DisposeAsync()
{
_runner.Dispose();
return Task.CompletedTask;
}
}
public Task DisposeAsync() => _mongo.DisposeAsync();
}

View File

@@ -0,0 +1,451 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.Metrics;
using System.Linq;
using System.Net.Http.Headers;
using System.Net.Http.Json;
using System.Text.Json;
using System.Threading;
using EphemeralMongo;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Concelier.RawModels;
using StellaOps.Excititor.WebService.Contracts;
using StellaOps.Ingestion.Telemetry;
using Xunit;
namespace StellaOps.Excititor.WebService.Tests;
public sealed class BatchIngestValidationTests : IDisposable
{
private const string Tenant = "tests";
private readonly IMongoRunner _runner;
private readonly TestWebApplicationFactory _factory;
public BatchIngestValidationTests()
{
_runner = MongoRunner.Run(new MongoRunnerOptions { UseSingleNodeReplicaSet = true });
_factory = new TestWebApplicationFactory(
configureConfiguration: configuration =>
{
configuration.AddInMemoryCollection(new Dictionary<string, string?>
{
["Excititor:Storage:Mongo:ConnectionString"] = _runner.ConnectionString,
["Excititor:Storage:Mongo:DatabaseName"] = "vex_batch_tests",
["Excititor:Storage:Mongo:DefaultTenant"] = Tenant,
});
},
configureServices: services =>
{
TestServiceOverrides.Apply(services);
services.AddTestAuthentication();
});
}
[Fact]
[Trait("Category", "BatchIngestValidation")]
public async Task BatchFixturesMaintainParityMetricsAndVerify()
{
using var metrics = new IngestionMetricListener();
using var client = CreateClient();
var fixtures = VexFixtureLibrary.CreateBatch();
foreach (var fixture in fixtures)
{
var ingestResponse = await client.PostAsJsonAsync("/ingest/vex", fixture.Request);
ingestResponse.EnsureSuccessStatusCode();
var payload = await ingestResponse.Content.ReadFromJsonAsync<VexIngestResponse>();
Assert.NotNull(payload);
fixture.RecordDigest(payload!.Digest);
}
var listResponse = await client.GetAsync($"/vex/raw?limit={fixtures.Count * 2}");
listResponse.EnsureSuccessStatusCode();
var listPayload = await listResponse.Content.ReadFromJsonAsync<VexRawListResponse>();
Assert.NotNull(listPayload);
foreach (var fixture in fixtures)
{
Assert.Contains(listPayload!.Records, record => record.Digest == fixture.Digest);
}
foreach (var fixture in fixtures)
{
var recordResponse = await client.GetAsync($"/vex/raw/{Uri.EscapeDataString(fixture.Digest)}");
recordResponse.EnsureSuccessStatusCode();
var record = await recordResponse.Content.ReadFromJsonAsync<VexRawRecordResponse>();
Assert.NotNull(record);
fixture.AssertRecord(record!);
}
var verifyRequest = new VexAocVerifyRequest(
DateTimeOffset.UtcNow.AddMinutes(-5),
DateTimeOffset.UtcNow.AddMinutes(5),
fixtures.Count + 5,
null,
null);
var verifyResponse = await client.PostAsJsonAsync("/aoc/verify", verifyRequest);
verifyResponse.EnsureSuccessStatusCode();
var verifyPayload = await verifyResponse.Content.ReadFromJsonAsync<VexAocVerifyResponse>();
Assert.NotNull(verifyPayload);
Assert.Equal(Tenant, verifyPayload!.Tenant);
Assert.Equal(fixtures.Count, verifyPayload.Checked.Vex);
Assert.Empty(verifyPayload.Violations);
Assert.Equal(fixtures.Count, verifyPayload.Metrics.IngestionWriteTotal);
Assert.Equal(0, verifyPayload.Metrics.AocViolationTotal);
Assert.False(verifyPayload.Truncated);
Assert.True(metrics.WaitForMeasurements(fixtures.Count, TimeSpan.FromSeconds(2)));
foreach (var measurement in metrics.GetMeasurements())
{
Assert.Equal(Tenant, measurement.Tenant);
Assert.Equal(IngestionTelemetry.ResultOk, measurement.Result);
Assert.Equal(1, measurement.Value);
}
}
private HttpClient CreateClient()
{
var client = _factory.CreateClient(new WebApplicationFactoryClientOptions
{
AllowAutoRedirect = false
});
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "vex.admin vex.read");
client.DefaultRequestHeaders.Add("X-Stella-Tenant", Tenant);
return client;
}
public void Dispose()
{
_factory.Dispose();
_runner.Dispose();
}
private sealed class IngestionMetricListener : IDisposable
{
private readonly List<Measurement> _measurements = new();
private readonly MeterListener _listener;
public IngestionMetricListener()
{
_listener = new MeterListener
{
InstrumentPublished = (instrument, listener) =>
{
if (instrument.Meter.Name == IngestionTelemetry.MeterName &&
instrument.Name == "ingestion_write_total")
{
listener.EnableMeasurementEvents(instrument);
}
}
};
_listener.SetMeasurementEventCallback<long>((instrument, measurement, tags, state) =>
{
if (instrument.Meter.Name != IngestionTelemetry.MeterName ||
instrument.Name != "ingestion_write_total")
{
return;
}
string tenant = string.Empty;
string source = string.Empty;
string result = string.Empty;
foreach (var tag in tags)
{
switch (tag.Key)
{
case "tenant":
tenant = tag.Value?.ToString() ?? string.Empty;
break;
case "source":
source = tag.Value?.ToString() ?? string.Empty;
break;
case "result":
result = tag.Value?.ToString() ?? string.Empty;
break;
}
}
lock (_measurements)
{
_measurements.Add(new Measurement(tenant, source, result, measurement));
}
});
_listener.Start();
}
public bool WaitForMeasurements(int expected, TimeSpan timeout)
{
var sw = Stopwatch.StartNew();
while (sw.Elapsed < timeout)
{
lock (_measurements)
{
if (_measurements.Count >= expected)
{
return true;
}
}
Thread.Sleep(25);
}
return false;
}
public IReadOnlyList<Measurement> GetMeasurements()
{
lock (_measurements)
{
return _measurements.ToList();
}
}
public void Dispose() => _listener.Dispose();
internal sealed record Measurement(string Tenant, string Source, string Result, long Value);
}
private sealed record VexFixture(
string Name,
VexIngestRequest Request,
string ExpectedFormat,
Action<JsonElement> ContentAssertion)
{
private string? _digest;
public string Digest => _digest ?? throw new InvalidOperationException("Digest not recorded yet.");
public void RecordDigest(string digest)
{
_digest = digest ?? throw new ArgumentNullException(nameof(digest));
}
public void AssertRecord(VexRawRecordResponse record)
{
Assert.Equal(ExpectedFormat, record.Document.Content.Format, StringComparer.OrdinalIgnoreCase);
ContentAssertion(record.Document.Content.Raw);
}
}
private static class VexFixtureLibrary
{
public static IReadOnlyList<VexFixture> CreateBatch()
=> new[]
{
CreateCycloneDxFixture("001", "sha256:batch-cdx-001", "CDX-BATCH-001", "not_affected"),
CreateCycloneDxFixture("002", "sha256:batch-cdx-002", "CDX-BATCH-002", "affected"),
CreateCycloneDxFixture("003", "sha256:batch-cdx-003", "CDX-BATCH-003", "fixed"),
CreateCsafFixture("010", "sha256:batch-csaf-001", "CSAF-BATCH-001", "fixed"),
CreateCsafFixture("011", "sha256:batch-csaf-002", "CSAF-BATCH-002", "known_affected"),
CreateCsafFixture("012", "sha256:batch-csaf-003", "CSAF-BATCH-003", "known_not_affected"),
CreateOpenVexFixture("020", "sha256:batch-openvex-001", "OVX-BATCH-001", "affected"),
CreateOpenVexFixture("021", "sha256:batch-openvex-002", "OVX-BATCH-002", "not_affected"),
CreateOpenVexFixture("022", "sha256:batch-openvex-003", "OVX-BATCH-003", "fixed")
};
private static VexFixture CreateCycloneDxFixture(string suffix, string digest, string upstreamId, string state)
{
var vulnerabilityId = $"CVE-2025-{suffix}";
var raw = $$"""
{
"bomFormat": "CycloneDX",
"specVersion": "1.6",
"version": 1,
"metadata": {
"timestamp": "2025-11-08T00:00:00Z",
"tools": [
{ "vendor": "stellaops", "name": "batch-cdx" }
]
},
"vulnerabilities": [
{
"id": "{{vulnerabilityId}}",
"analysis": { "state": "{{state}}" },
"ratings": [
{ "score": 0.0, "method": "cvssv3" }
]
}
]
}
""";
return new VexFixture(
$"cyclonedx-{suffix}",
BuildRequest(
providerId: $"cyclonedx:batch:{suffix}",
vendor: "vendor:cyclonedx",
connector: "cdx-batch",
stream: "cyclonedx-vex",
format: "cyclonedx",
specVersion: "1.6",
rawJson: raw,
digest: digest,
upstreamId: upstreamId,
sourceUri: $"https://example.test/vex/cyclonedx/{suffix}"),
"cyclonedx",
element =>
{
var actual = element
.GetProperty("vulnerabilities")[0]
.GetProperty("analysis")
.GetProperty("state")
.GetString();
Assert.Equal(state, actual);
});
}
private static VexFixture CreateCsafFixture(string suffix, string digest, string upstreamId, string statusKey)
{
var cve = $"CVE-2025-{suffix}";
var productId = $"csaf-prod-{suffix}";
var raw = $$"""
{
"document": {
"category": "csaf_vex",
"title": "Sample CSAF VEX",
"tracking": {
"id": "CSAF-2025-{{suffix}}",
"version": "1",
"current_release_date": "2025-11-07T00:00:00Z",
"initial_release_date": "2025-11-07T00:00:00Z",
"status": "final"
}
},
"product_tree": {
"branches": [
{
"name": "products",
"product": {
"name": "sample-product-{{suffix}}",
"product_id": "{{productId}}"
}
}
]
},
"vulnerabilities": [
{
"cve": "{{cve}}",
"product_status": {
"{{statusKey}}": [ "{{productId}}" ]
},
"threats": [
{ "category": "impact", "details": "none" }
]
}
]
}
""";
return new VexFixture(
$"csaf-{suffix}",
BuildRequest(
providerId: $"csaf:batch:{suffix}",
vendor: "vendor:csaf",
connector: "csaf-batch",
stream: "csaf-vex",
format: "csaf",
specVersion: "2.0",
rawJson: raw,
digest: digest,
upstreamId: upstreamId,
sourceUri: $"https://example.test/vex/csaf/{suffix}"),
"csaf",
element =>
{
var productStatus = element
.GetProperty("vulnerabilities")[0]
.GetProperty("product_status")
.GetProperty(statusKey)
.EnumerateArray()
.First()
.GetString();
Assert.Equal(productId, productStatus);
});
}
private static VexFixture CreateOpenVexFixture(string suffix, string digest, string upstreamId, string status)
{
var raw = $$"""
{
"context": "https://openvex.dev/ns/v0.2.0",
"statements": [
{
"vulnerability": "CVE-2025-{{suffix}}",
"products": [
"pkg:docker/demo@sha256:{{digest}}"
],
"status": "{{status}}",
"statusNotes": "waiting on vendor patch"
}
]
}
""";
return new VexFixture(
$"openvex-{suffix}",
BuildRequest(
providerId: $"openvex:batch:{suffix}",
vendor: "vendor:openvex",
connector: "openvex-batch",
stream: "openvex",
format: "openvex",
specVersion: "0.2.0",
rawJson: raw,
digest: digest,
upstreamId: upstreamId,
sourceUri: $"https://example.test/vex/openvex/{suffix}"),
"openvex",
element =>
{
var actual = element
.GetProperty("statements")[0]
.GetProperty("status")
.GetString();
Assert.Equal(status, actual);
});
}
private static VexIngestRequest BuildRequest(
string providerId,
string vendor,
string connector,
string stream,
string format,
string specVersion,
string rawJson,
string digest,
string upstreamId,
string sourceUri)
{
using var rawDocument = JsonDocument.Parse(rawJson);
var metadata = new Dictionary<string, string>
{
["source.vendor"] = vendor,
["source.connector"] = connector,
["source.stream"] = stream,
["source.connector_version"] = "1.0.0"
};
return new VexIngestRequest(
providerId,
new VexIngestSourceRequest(vendor, connector, "1.0.0", stream),
new VexIngestUpstreamRequest(
sourceUri,
upstreamId,
"1",
DateTimeOffset.UtcNow,
digest,
new VexIngestSignatureRequest(false, null, null, null, null, null),
new Dictionary<string, string>()),
new VexIngestContentRequest(format, specVersion, rawDocument.RootElement.Clone(), null),
metadata);
}
}
}

View File

@@ -0,0 +1,208 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using EphemeralMongo;
using MongoDB.Bson;
using MongoDB.Driver;
using Xunit;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.WebService.Services;
namespace StellaOps.Excititor.WebService.Tests;
public sealed class ObservabilityEndpointTests : IDisposable
{
private readonly TestWebApplicationFactory _factory;
private readonly IMongoRunner _runner;
public ObservabilityEndpointTests()
{
_runner = MongoRunner.Run(new MongoRunnerOptions { UseSingleNodeReplicaSet = true });
_factory = new TestWebApplicationFactory(
configureConfiguration: configuration =>
{
configuration.AddInMemoryCollection(new Dictionary<string, string?>
{
["Excititor:Storage:Mongo:ConnectionString"] = _runner.ConnectionString,
["Excititor:Storage:Mongo:DatabaseName"] = "excititor_obs_tests",
["Excititor:Storage:Mongo:RawBucketName"] = "vex.raw",
["Excititor:Observability:IngestWarningThreshold"] = "00:10:00",
["Excititor:Observability:IngestCriticalThreshold"] = "00:30:00",
["Excititor:Observability:SignatureWindow"] = "00:30:00",
["Excititor:Observability:ConflictTrendWindow"] = "01:00:00",
["Excititor:Observability:ConflictTrendBucketMinutes"] = "5"
});
},
configureServices: services =>
{
TestServiceOverrides.Apply(services);
services.AddTestAuthentication();
services.RemoveAll<IVexConnectorStateRepository>();
services.AddScoped<IVexConnectorStateRepository, MongoVexConnectorStateRepository>();
services.AddSingleton<IVexConnector>(_ => new StubConnector("excititor:redhat", VexProviderKind.Distro));
});
SeedDatabase();
}
[Fact]
public async Task HealthEndpoint_ReturnsAggregatedMetrics()
{
var client = _factory.CreateClient(new WebApplicationFactoryClientOptions
{
AllowAutoRedirect = false
});
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "vex.admin");
using var response = await client.GetAsync("/obs/excititor/health");
var payload = await response.Content.ReadAsStringAsync();
Assert.True(response.IsSuccessStatusCode, payload);
using var document = JsonDocument.Parse(payload);
var root = document.RootElement;
var ingest = root.GetProperty("ingest");
Assert.Equal("healthy", ingest.GetProperty("status").GetString());
var connectors = ingest.GetProperty("connectors");
Assert.Equal(1, connectors.GetArrayLength());
Assert.Equal("excititor:redhat", connectors[0].GetProperty("connectorId").GetString());
var signature = root.GetProperty("signature");
Assert.Equal(3, signature.GetProperty("documentsEvaluated").GetInt32());
Assert.Equal(1, signature.GetProperty("failures").GetInt32());
Assert.Equal(1, signature.GetProperty("verified").GetInt32());
var conflicts = root.GetProperty("conflicts");
Assert.True(conflicts.GetProperty("conflictStatements").GetInt64() >= 2);
Assert.True(conflicts.GetProperty("trend").GetArrayLength() >= 1);
}
private void SeedDatabase()
{
using var scope = _factory.Services.CreateScope();
var database = scope.ServiceProvider.GetRequiredService<IMongoDatabase>();
database.DropCollection(VexMongoCollectionNames.Raw);
database.DropCollection(VexMongoCollectionNames.Consensus);
database.DropCollection(VexMongoCollectionNames.ConnectorState);
var now = DateTime.UtcNow;
var rawCollection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
rawCollection.InsertMany(new[]
{
new BsonDocument
{
{ "Id", "raw-1" },
{ "ProviderId", "excititor:redhat" },
{ ObservabilityEndpointTestsHelper.RetrievedAtField, now },
{ ObservabilityEndpointTestsHelper.MetadataField, new BsonDocument { { "signature.present", "true" }, { "signature.verified", "true" } } }
},
new BsonDocument
{
{ "Id", "raw-2" },
{ "ProviderId", "excititor:redhat" },
{ ObservabilityEndpointTestsHelper.RetrievedAtField, now },
{ ObservabilityEndpointTestsHelper.MetadataField, new BsonDocument { { "signature.present", "true" } } }
},
new BsonDocument
{
{ "Id", "raw-3" },
{ "ProviderId", "excititor:redhat" },
{ ObservabilityEndpointTestsHelper.RetrievedAtField, now },
{ ObservabilityEndpointTestsHelper.MetadataField, new BsonDocument() }
}
});
var consensus = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Consensus);
consensus.InsertMany(new[]
{
ObservabilityEndpointTestsHelper.CreateConsensusDocument("c1", now, "affected"),
ObservabilityEndpointTestsHelper.CreateConsensusDocument("c2", now.AddMinutes(-5), "not_affected")
});
var stateRepository = scope.ServiceProvider.GetRequiredService<IVexConnectorStateRepository>();
var state = new VexConnectorState(
"excititor:redhat",
now.AddMinutes(-5),
ImmutableArray<string>.Empty,
ImmutableDictionary<string, string>.Empty,
now.AddMinutes(-5),
0,
now.AddMinutes(10),
null);
stateRepository.SaveAsync(state, CancellationToken.None).GetAwaiter().GetResult();
}
public void Dispose()
{
_factory.Dispose();
_runner.Dispose();
}
private sealed class StubConnector : IVexConnector
{
public StubConnector(string id, VexProviderKind kind)
{
Id = id;
Kind = kind;
}
public string Id { get; }
public VexProviderKind Kind { get; }
public ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
=> ValueTask.CompletedTask;
public IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, CancellationToken cancellationToken)
=> AsyncEnumerable.Empty<VexRawDocument>();
public ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
=> ValueTask.FromResult(new VexClaimBatch(
document,
ImmutableArray<VexClaim>.Empty,
ImmutableDictionary<string, string>.Empty));
}
}
internal static class ObservabilityEndpointTestsHelper
{
public const string RetrievedAtField = "RetrievedAt";
public const string MetadataField = "Metadata";
public static BsonDocument CreateConsensusDocument(string id, DateTime timestamp, string conflictStatus)
{
var conflicts = new BsonArray
{
new BsonDocument
{
{ "ProviderId", "excititor:redhat" },
{ "Status", conflictStatus },
{ "DocumentDigest", Guid.NewGuid().ToString("n") }
}
};
return new BsonDocument
{
{ "Id", id },
{ "VulnerabilityId", $"CVE-{id}" },
{ "Product", new BsonDocument { { "Key", $"pkg:{id}" }, { "Name", $"pkg-{id}" } } },
{ "Status", "affected" },
{ "CalculatedAt", timestamp },
{ "Conflicts", conflicts }
};
}
}

View File

@@ -19,8 +19,9 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../../StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj" />
<ProjectReference Include="../../../Aoc/__Libraries/StellaOps.Aoc/StellaOps.Aoc.csproj" />
</ItemGroup>
<ItemGroup>
<Using Include="Xunit" />
</ItemGroup>
</Project>
</Project>

View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Immutable;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Text;
using System.Text.Json;
using Microsoft.Extensions.DependencyInjection;
@@ -9,11 +10,12 @@ using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Attestation.Verification;
using StellaOps.Excititor.Export;
using StellaOps.Excititor.Export;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.WebService.Services;
using MongoDB.Driver;
using StellaOps.Excititor.Attestation.Dsse;
using StellaOps.Excititor.Attestation.Dsse;
using StellaOps.Excititor.Attestation.Signing;
namespace StellaOps.Excititor.WebService.Tests;
@@ -24,23 +26,25 @@ internal static class TestServiceOverrides
services.RemoveAll<IVexConnector>();
services.RemoveAll<IVexIngestOrchestrator>();
services.RemoveAll<IVexConnectorStateRepository>();
services.RemoveAll<IVexExportCacheService>();
services.RemoveAll<IVexExportDataSource>();
services.RemoveAll<IVexExportStore>();
services.RemoveAll<IVexCacheIndex>();
services.RemoveAll<IVexCacheMaintenance>();
services.RemoveAll<IVexAttestationClient>();
services.AddSingleton<IVexIngestOrchestrator, StubIngestOrchestrator>();
services.AddSingleton<IVexConnectorStateRepository, StubConnectorStateRepository>();
services.AddSingleton<IVexExportCacheService, StubExportCacheService>();
services.RemoveAll<IVexExportCacheService>();
services.RemoveAll<IVexExportDataSource>();
services.RemoveAll<IVexExportStore>();
services.RemoveAll<IVexCacheIndex>();
services.RemoveAll<IVexCacheMaintenance>();
services.RemoveAll<IVexAttestationClient>();
services.RemoveAll<IVexSigner>();
services.AddSingleton<IVexIngestOrchestrator, StubIngestOrchestrator>();
services.AddSingleton<IVexConnectorStateRepository, StubConnectorStateRepository>();
services.AddSingleton<IVexExportCacheService, StubExportCacheService>();
services.RemoveAll<IExportEngine>();
services.AddSingleton<IExportEngine, StubExportEngine>();
services.AddSingleton<IVexExportDataSource, StubExportDataSource>();
services.AddSingleton<IVexExportStore, StubExportStore>();
services.AddSingleton<IVexCacheIndex, StubCacheIndex>();
services.AddSingleton<IVexCacheMaintenance, StubCacheMaintenance>();
services.AddSingleton<IVexAttestationClient, StubAttestationClient>();
services.AddSingleton<IVexExportStore, StubExportStore>();
services.AddSingleton<IVexCacheIndex, StubCacheIndex>();
services.AddSingleton<IVexCacheMaintenance, StubCacheMaintenance>();
services.AddSingleton<IVexAttestationClient, StubAttestationClient>();
services.AddSingleton<IVexSigner, StubSigner>();
services.RemoveAll<IHostedService>();
services.AddSingleton<IHostedService, NoopHostedService>();
@@ -135,8 +139,8 @@ internal static class TestServiceOverrides
=> ValueTask.FromResult(0);
}
private sealed class StubAttestationClient : IVexAttestationClient
{
private sealed class StubAttestationClient : IVexAttestationClient
{
public ValueTask<VexAttestationResponse> SignAsync(VexAttestationRequest request, CancellationToken cancellationToken)
{
var envelope = new DsseEnvelope(
@@ -168,22 +172,34 @@ internal static class TestServiceOverrides
}
}
private sealed class StubConnectorStateRepository : IVexConnectorStateRepository
{
private readonly ConcurrentDictionary<string, VexConnectorState> _states = new(StringComparer.Ordinal);
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
private sealed class StubConnectorStateRepository : IVexConnectorStateRepository
{
private readonly ConcurrentDictionary<string, VexConnectorState> _states = new(StringComparer.Ordinal);
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
_states.TryGetValue(connectorId, out var state);
return ValueTask.FromResult(state);
}
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
_states[state.ConnectorId] = state;
return ValueTask.CompletedTask;
}
}
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
_states[state.ConnectorId] = state;
return ValueTask.CompletedTask;
}
public ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
IReadOnlyCollection<VexConnectorState> snapshot = _states.Values.ToList();
return ValueTask.FromResult(snapshot);
}
}
private sealed class StubSigner : IVexSigner
{
public ValueTask<VexSignedPayload> SignAsync(ReadOnlyMemory<byte> payload, CancellationToken cancellationToken)
=> ValueTask.FromResult(new VexSignedPayload("stub-signature", "stub-key"));
}
private sealed class StubIngestOrchestrator : IVexIngestOrchestrator
{

View File

@@ -0,0 +1,203 @@
using System;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Nodes;
using StellaOps.Aoc;
namespace StellaOps.Excititor.WebService.Tests;
public sealed class VexGuardSchemaTests
{
private static readonly AocWriteGuard Guard = new();
[Fact]
public void CycloneDxFixture_CompliesWithGuard()
{
var result = ValidateCycloneDx();
Assert.True(result.IsValid, DescribeViolations(result));
}
[Fact]
public void CsafFixture_CompliesWithGuard()
{
var result = ValidateCsaf();
Assert.True(result.IsValid, DescribeViolations(result));
}
[Fact]
public void CycloneDxFixture_WithForbiddenField_ProducesErrAoc001()
{
var result = ValidateCycloneDx(node => node["severity"] = "critical");
AssertViolation(result, "ERR_AOC_001", "/severity");
}
[Fact]
public void CycloneDxFixture_WithDerivedField_ProducesErrAoc006()
{
var result = ValidateCycloneDx(node => node["effective_owner"] = "security");
AssertViolation(result, "ERR_AOC_006", "/effective_owner");
}
[Fact]
public void CycloneDxFixture_WithUnknownField_ProducesErrAoc007()
{
var result = ValidateCycloneDx(node => node["custom_field"] = 123);
AssertViolation(result, "ERR_AOC_007", "/custom_field");
}
[Fact]
public void CycloneDxFixture_WithSupersedes_RemainsValid()
{
var result = ValidateCycloneDx(node => node["supersedes"] = "digest:prev-cdx");
Assert.True(result.IsValid, DescribeViolations(result));
}
[Fact]
public void CsafFixture_WithSupersedes_RemainsValid()
{
var result = ValidateCsaf(node => node["supersedes"] = "digest:prev-csaf");
Assert.True(result.IsValid, DescribeViolations(result));
}
private static AocGuardResult ValidateCycloneDx(Action<JsonObject>? mutate = null)
=> ValidateFixture(CycloneDxRaw, mutate);
private static AocGuardResult ValidateCsaf(Action<JsonObject>? mutate = null)
=> ValidateFixture(CsafRaw, mutate);
private static AocGuardResult ValidateFixture(string json, Action<JsonObject>? mutate)
{
var node = JsonNode.Parse(json)!.AsObject();
mutate?.Invoke(node);
using var document = JsonDocument.Parse(node.ToJsonString());
return Guard.Validate(document.RootElement);
}
private static void AssertViolation(AocGuardResult result, string expectedCode, string expectedPath)
{
Assert.False(result.IsValid);
Assert.Contains(result.Violations, violation =>
violation.ErrorCode == expectedCode && string.Equals(violation.Path, expectedPath, StringComparison.OrdinalIgnoreCase));
}
private static string DescribeViolations(AocGuardResult result)
=> string.Join(", ", result.Violations.Select(v => $"{v.ErrorCode}:{v.Path}"));
private const string CycloneDxRaw = """
{
"tenant": "tests",
"source": {
"vendor": "cyclonedx",
"connector": "cdx",
"version": "1.0.0",
"stream": "vex-cyclonedx"
},
"upstream": {
"upstream_id": "CDX-2025-0001",
"document_version": "2025.11.08",
"retrieved_at": "2025-11-08T00:00:00Z",
"content_hash": "sha256:cdx",
"signature": { "present": false }
},
"content": {
"format": "CycloneDX",
"spec_version": "1.6",
"raw": {
"bomFormat": "CycloneDX",
"specVersion": "1.6",
"serialNumber": "urn:uuid:12345678-1234-5678-9abc-def012345678",
"version": 1,
"metadata": {
"timestamp": "2025-11-08T00:00:00Z",
"tools": [
{ "vendor": "stellaops", "name": "sample-vex-bot" }
]
},
"vulnerabilities": [
{
"id": "CVE-2025-0001",
"analysis": { "state": "not_affected" },
"ratings": [
{ "score": 0.0, "method": "cvssv3" }
]
}
]
}
},
"linkset": {
"aliases": [],
"references": [],
"relationships": [],
"products": [],
"notes": {},
"reconciled_from": []
}
}
""";
private const string CsafRaw = """
{
"tenant": "tests",
"source": {
"vendor": "csaf",
"connector": "csaf-json",
"version": "1.2.3",
"stream": "vex-csaf"
},
"upstream": {
"upstream_id": "CSAF-2025-0002",
"document_version": "2025.11.07",
"retrieved_at": "2025-11-08T01:10:00Z",
"content_hash": "sha256:csaf",
"signature": { "present": false }
},
"content": {
"format": "CSAF",
"spec_version": "2.0",
"raw": {
"document": {
"category": "csaf_vex",
"title": "Sample CSAF VEX",
"tracking": {
"id": "CSAF-2025-0002",
"version": "1",
"current_release_date": "2025-11-07T00:00:00Z",
"initial_release_date": "2025-11-07T00:00:00Z",
"status": "final"
}
},
"product_tree": {
"branches": [
{
"name": "products",
"product": {
"name": "sample-product",
"product_id": "csaf-prod"
}
}
]
},
"vulnerabilities": [
{
"cve": "CVE-2025-0002",
"product_status": {
"fixed": [ "csaf-prod" ]
},
"threats": [
{ "category": "impact", "details": "none" }
]
}
]
}
},
"linkset": {
"aliases": [],
"references": [],
"relationships": [],
"products": [],
"notes": {},
"reconciled_from": []
}
}
""";
}

View File

@@ -0,0 +1,107 @@
using System;
using System.Collections.Generic;
using System.Net.Http.Headers;
using System.Net.Http.Json;
using System.Text.Json;
using EphemeralMongo;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.WebService.Contracts;
using Xunit;
namespace StellaOps.Excititor.WebService.Tests;
public sealed class VexRawEndpointsTests : IDisposable
{
private readonly IMongoRunner _runner;
private readonly TestWebApplicationFactory _factory;
public VexRawEndpointsTests()
{
_runner = MongoRunner.Run(new MongoRunnerOptions { UseSingleNodeReplicaSet = true });
_factory = new TestWebApplicationFactory(
configureConfiguration: configuration =>
{
configuration.AddInMemoryCollection(new Dictionary<string, string?>
{
["Excititor:Storage:Mongo:ConnectionString"] = _runner.ConnectionString,
["Excititor:Storage:Mongo:DatabaseName"] = "vex_raw_tests",
["Excititor:Storage:Mongo:DefaultTenant"] = "tests",
});
},
configureServices: services =>
{
TestServiceOverrides.Apply(services);
services.AddTestAuthentication();
});
}
[Fact]
public async Task IngestListGetAndVerifyFlow()
{
using var client = _factory.CreateClient(new WebApplicationFactoryClientOptions
{
AllowAutoRedirect = false
});
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "vex.admin vex.read");
client.DefaultRequestHeaders.Add("X-Stella-Tenant", "tests");
var ingestRequest = BuildVexIngestRequest();
var ingestResponse = await client.PostAsJsonAsync("/ingest/vex", ingestRequest);
ingestResponse.EnsureSuccessStatusCode();
var ingestPayload = await ingestResponse.Content.ReadFromJsonAsync<VexIngestResponse>();
Assert.NotNull(ingestPayload);
Assert.True(ingestPayload!.Inserted);
var getResponse = await client.GetAsync($"/vex/raw/{Uri.EscapeDataString(ingestPayload.Digest)}");
getResponse.EnsureSuccessStatusCode();
var record = await getResponse.Content.ReadFromJsonAsync<VexRawRecordResponse>();
Assert.NotNull(record);
Assert.Equal(ingestPayload.Digest, record!.Digest);
var listResponse = await client.GetAsync("/vex/raw?limit=5");
listResponse.EnsureSuccessStatusCode();
var listPayload = await listResponse.Content.ReadFromJsonAsync<VexRawListResponse>();
Assert.NotNull(listPayload);
Assert.Contains(listPayload!.Records, summary => summary.Digest == ingestPayload.Digest);
var verifyRequest = new VexAocVerifyRequest(null, null, 10, null, null);
var verifyResponse = await client.PostAsJsonAsync("/aoc/verify", verifyRequest);
verifyResponse.EnsureSuccessStatusCode();
var verifyPayload = await verifyResponse.Content.ReadFromJsonAsync<VexAocVerifyResponse>();
Assert.NotNull(verifyPayload);
Assert.True(verifyPayload!.Checked.Vex >= 1);
}
private static VexIngestRequest BuildVexIngestRequest()
{
using var contentDocument = JsonDocument.Parse("{\"vex\":\"payload\"}");
return new VexIngestRequest(
ProviderId: "excititor:test",
Source: new VexIngestSourceRequest("vendor:test", "connector:test", "1.0.0", "csaf"),
Upstream: new VexIngestUpstreamRequest(
SourceUri: "https://example.test/vex.json",
UpstreamId: "VEX-TEST-001",
DocumentVersion: "1",
RetrievedAt: DateTimeOffset.UtcNow,
ContentHash: "sha256:test",
Signature: new VexIngestSignatureRequest(false, null, null, null, null, null),
Provenance: new Dictionary<string, string>()),
Content: new VexIngestContentRequest("csaf", "2.0", contentDocument.RootElement.Clone(), null),
Metadata: new Dictionary<string, string>
{
["source.vendor"] = "vendor:test"
});
}
public void Dispose()
{
_factory.Dispose();
_runner.Dispose();
}
}