Add SBOM, symbols, traces, and VEX files for CVE-2022-21661 SQLi case
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Created CycloneDX and SPDX SBOM files for both reachable and unreachable images. - Added symbols.json detailing function entry and sink points in the WordPress code. - Included runtime traces for function calls in both reachable and unreachable scenarios. - Developed OpenVEX files indicating vulnerability status and justification for both cases. - Updated README for evaluator harness to guide integration with scanner output.
This commit is contained in:
@@ -6,6 +6,7 @@ ASP.NET Minimal API surface for Excititor ingest, provider administration, recon
|
||||
- HTTP endpoints `/excititor/*` with authentication, authorization scopes, request validation, and deterministic responses.
|
||||
- Job orchestration bridges for Worker hand-off (when co-hosted) and offline-friendly configuration.
|
||||
- Observability (structured logs, metrics, tracing) aligned with StellaOps conventions.
|
||||
- Optional/minor DI dependencies on minimal APIs must be declared with `[FromServices] SomeType? service = null` parameters so endpoint tests do not require bespoke service registrations.
|
||||
## Participants
|
||||
- StellaOps.Cli sends `excititor` verbs to this service via token-authenticated HTTPS.
|
||||
- Worker receives scheduled jobs and uses shared infrastructure via common DI extensions.
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using StellaOps.Concelier.RawModels;
|
||||
using RawVexDocumentModel = StellaOps.Concelier.RawModels.VexRawDocument;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Contracts;
|
||||
|
||||
public sealed record VexIngestRequest(
|
||||
[property: JsonPropertyName("providerId")] string ProviderId,
|
||||
[property: JsonPropertyName("source")] VexIngestSourceRequest Source,
|
||||
[property: JsonPropertyName("upstream")] VexIngestUpstreamRequest Upstream,
|
||||
[property: JsonPropertyName("content")] VexIngestContentRequest Content,
|
||||
[property: JsonPropertyName("metadata")] IReadOnlyDictionary<string, string>? Metadata);
|
||||
|
||||
public sealed record VexIngestSourceRequest(
|
||||
[property: JsonPropertyName("vendor")] string Vendor,
|
||||
[property: JsonPropertyName("connector")] string Connector,
|
||||
[property: JsonPropertyName("version")] string Version,
|
||||
[property: JsonPropertyName("stream")] string? Stream);
|
||||
|
||||
public sealed record VexIngestUpstreamRequest(
|
||||
[property: JsonPropertyName("sourceUri")] string SourceUri,
|
||||
[property: JsonPropertyName("upstreamId")] string UpstreamId,
|
||||
[property: JsonPropertyName("documentVersion")] string? DocumentVersion,
|
||||
[property: JsonPropertyName("retrievedAt")] DateTimeOffset? RetrievedAt,
|
||||
[property: JsonPropertyName("contentHash")] string? ContentHash,
|
||||
[property: JsonPropertyName("signature")] VexIngestSignatureRequest? Signature,
|
||||
[property: JsonPropertyName("provenance")] IReadOnlyDictionary<string, string>? Provenance);
|
||||
|
||||
public sealed record VexIngestSignatureRequest(
|
||||
[property: JsonPropertyName("present")] bool Present,
|
||||
[property: JsonPropertyName("format")] string? Format,
|
||||
[property: JsonPropertyName("keyId")] string? KeyId,
|
||||
[property: JsonPropertyName("sig")] string? Signature,
|
||||
[property: JsonPropertyName("certificate")] string? Certificate,
|
||||
[property: JsonPropertyName("digest")] string? Digest);
|
||||
|
||||
public sealed record VexIngestContentRequest(
|
||||
[property: JsonPropertyName("format")] string Format,
|
||||
[property: JsonPropertyName("specVersion")] string? SpecVersion,
|
||||
[property: JsonPropertyName("raw")] JsonElement Raw,
|
||||
[property: JsonPropertyName("encoding")] string? Encoding);
|
||||
|
||||
public sealed record VexIngestResponse(
|
||||
[property: JsonPropertyName("digest")] string Digest,
|
||||
[property: JsonPropertyName("inserted")] bool Inserted,
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("retrievedAt")] DateTimeOffset RetrievedAt);
|
||||
|
||||
public sealed record VexRawSummaryResponse(
|
||||
[property: JsonPropertyName("digest")] string Digest,
|
||||
[property: JsonPropertyName("providerId")] string ProviderId,
|
||||
[property: JsonPropertyName("format")] string Format,
|
||||
[property: JsonPropertyName("sourceUri")] string SourceUri,
|
||||
[property: JsonPropertyName("retrievedAt")] DateTimeOffset RetrievedAt,
|
||||
[property: JsonPropertyName("inlineContent")] bool InlineContent,
|
||||
[property: JsonPropertyName("metadata")] IReadOnlyDictionary<string, string> Metadata);
|
||||
|
||||
public sealed record VexRawListResponse(
|
||||
[property: JsonPropertyName("records")] IReadOnlyList<VexRawSummaryResponse> Records,
|
||||
[property: JsonPropertyName("nextCursor")] string? NextCursor,
|
||||
[property: JsonPropertyName("hasMore")] bool HasMore);
|
||||
|
||||
public sealed record VexRawRecordResponse(
|
||||
[property: JsonPropertyName("digest")] string Digest,
|
||||
[property: JsonPropertyName("document")] RawVexDocumentModel Document,
|
||||
[property: JsonPropertyName("retrievedAt")] DateTimeOffset RetrievedAt);
|
||||
|
||||
public sealed record VexRawProvenanceResponse(
|
||||
[property: JsonPropertyName("digest")] string Digest,
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("source")] RawSourceMetadata Source,
|
||||
[property: JsonPropertyName("upstream")] RawUpstreamMetadata Upstream,
|
||||
[property: JsonPropertyName("retrievedAt")] DateTimeOffset RetrievedAt);
|
||||
|
||||
public sealed record VexAocVerifyRequest(
|
||||
[property: JsonPropertyName("since")] DateTimeOffset? Since,
|
||||
[property: JsonPropertyName("until")] DateTimeOffset? Until,
|
||||
[property: JsonPropertyName("limit")] int? Limit,
|
||||
[property: JsonPropertyName("sources")] IReadOnlyList<string>? Sources,
|
||||
[property: JsonPropertyName("codes")] IReadOnlyList<string>? Codes);
|
||||
|
||||
public sealed record VexAocVerifyResponse(
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("window")] VexAocVerifyWindow Window,
|
||||
[property: JsonPropertyName("checked")] VexAocVerifyChecked Checked,
|
||||
[property: JsonPropertyName("violations")] IReadOnlyList<VexAocVerifyViolation> Violations,
|
||||
[property: JsonPropertyName("metrics")] VexAocVerifyMetrics Metrics,
|
||||
[property: JsonPropertyName("truncated")] bool Truncated);
|
||||
|
||||
public sealed record VexAocVerifyWindow(
|
||||
[property: JsonPropertyName("from")] DateTimeOffset From,
|
||||
[property: JsonPropertyName("to")] DateTimeOffset To);
|
||||
|
||||
public sealed record VexAocVerifyChecked(
|
||||
[property: JsonPropertyName("advisories")] int Advisories,
|
||||
[property: JsonPropertyName("vex")] int Vex);
|
||||
|
||||
public sealed record VexAocVerifyMetrics(
|
||||
[property: JsonPropertyName("ingestion_write_total")] int IngestionWriteTotal,
|
||||
[property: JsonPropertyName("aoc_violation_total")] int AocViolationTotal);
|
||||
|
||||
public sealed record VexAocVerifyViolation(
|
||||
[property: JsonPropertyName("code")] string Code,
|
||||
[property: JsonPropertyName("count")] int Count,
|
||||
[property: JsonPropertyName("examples")] IReadOnlyList<VexAocVerifyViolationExample> Examples);
|
||||
|
||||
public sealed record VexAocVerifyViolationExample(
|
||||
[property: JsonPropertyName("source")] string Source,
|
||||
[property: JsonPropertyName("documentId")] string DocumentId,
|
||||
[property: JsonPropertyName("contentHash")] string ContentHash,
|
||||
[property: JsonPropertyName("path")] string Path);
|
||||
@@ -0,0 +1,76 @@
|
||||
using System.Diagnostics;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.Primitives;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Extensions;
|
||||
|
||||
internal static class ObservabilityExtensions
|
||||
{
|
||||
private const string TraceHeaderName = "X-Stella-TraceId";
|
||||
private const string CorrelationHeaderName = "X-Stella-CorrelationId";
|
||||
private const string LegacyCorrelationHeaderName = "X-Correlation-Id";
|
||||
private const string CorrelationItemKey = "__stella.correlationId";
|
||||
|
||||
public static IApplicationBuilder UseObservabilityHeaders(this IApplicationBuilder app)
|
||||
{
|
||||
return app.Use((context, next) =>
|
||||
{
|
||||
var correlationId = ResolveCorrelationId(context);
|
||||
context.Items[CorrelationItemKey] = correlationId;
|
||||
|
||||
context.Response.OnStarting(state =>
|
||||
{
|
||||
var httpContext = (HttpContext)state;
|
||||
ApplyHeaders(httpContext);
|
||||
return Task.CompletedTask;
|
||||
}, context);
|
||||
|
||||
return next();
|
||||
});
|
||||
}
|
||||
|
||||
private static void ApplyHeaders(HttpContext context)
|
||||
{
|
||||
var traceId = Activity.Current?.TraceId.ToString() ?? context.TraceIdentifier;
|
||||
if (!string.IsNullOrWhiteSpace(traceId))
|
||||
{
|
||||
context.Response.Headers[TraceHeaderName] = traceId;
|
||||
}
|
||||
|
||||
var correlationId = ResolveCorrelationId(context);
|
||||
if (!string.IsNullOrWhiteSpace(correlationId))
|
||||
{
|
||||
context.Response.Headers[CorrelationHeaderName] = correlationId!;
|
||||
}
|
||||
}
|
||||
|
||||
private static string ResolveCorrelationId(HttpContext context)
|
||||
{
|
||||
if (context.Items.TryGetValue(CorrelationItemKey, out var existing) && existing is string cached && !string.IsNullOrWhiteSpace(cached))
|
||||
{
|
||||
return cached;
|
||||
}
|
||||
|
||||
if (TryReadHeader(context.Request.Headers, CorrelationHeaderName, out var headerValue) ||
|
||||
TryReadHeader(context.Request.Headers, LegacyCorrelationHeaderName, out headerValue))
|
||||
{
|
||||
return headerValue!;
|
||||
}
|
||||
|
||||
return context.TraceIdentifier;
|
||||
}
|
||||
|
||||
private static bool TryReadHeader(IHeaderDictionary headers, string name, out string? value)
|
||||
{
|
||||
if (headers.TryGetValue(name, out StringValues header) && !StringValues.IsNullOrEmpty(header))
|
||||
{
|
||||
value = header.ToString();
|
||||
return true;
|
||||
}
|
||||
|
||||
value = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Reflection;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using OpenTelemetry.Metrics;
|
||||
using OpenTelemetry.Resources;
|
||||
using OpenTelemetry.Trace;
|
||||
using StellaOps.Excititor.WebService.Options;
|
||||
using StellaOps.Ingestion.Telemetry;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Extensions;
|
||||
|
||||
internal static class TelemetryExtensions
|
||||
{
|
||||
public static void ConfigureExcititorTelemetry(this WebApplicationBuilder builder)
|
||||
{
|
||||
var telemetryOptions = new ExcititorTelemetryOptions();
|
||||
builder.Configuration.GetSection("Excititor:Telemetry").Bind(telemetryOptions);
|
||||
|
||||
if (!telemetryOptions.Enabled || (!telemetryOptions.EnableTracing && !telemetryOptions.EnableMetrics))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var openTelemetry = builder.Services.AddOpenTelemetry();
|
||||
|
||||
openTelemetry.ConfigureResource(resource =>
|
||||
{
|
||||
var serviceName = telemetryOptions.ServiceName ?? builder.Environment.ApplicationName ?? "StellaOps.Excititor.WebService";
|
||||
var version = Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown";
|
||||
resource.AddService(serviceName, serviceVersion: version, serviceInstanceId: Environment.MachineName);
|
||||
|
||||
foreach (var attribute in telemetryOptions.ResourceAttributes)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(attribute.Key) || string.IsNullOrWhiteSpace(attribute.Value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
resource.AddAttributes(new[]
|
||||
{
|
||||
new KeyValuePair<string, object>(attribute.Key, attribute.Value)
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
if (telemetryOptions.EnableTracing)
|
||||
{
|
||||
openTelemetry.WithTracing(tracing =>
|
||||
{
|
||||
tracing
|
||||
.AddSource(IngestionTelemetry.ActivitySourceName)
|
||||
.AddAspNetCoreInstrumentation()
|
||||
.AddHttpClientInstrumentation();
|
||||
|
||||
ConfigureExporters(telemetryOptions, tracing);
|
||||
});
|
||||
}
|
||||
|
||||
if (telemetryOptions.EnableMetrics)
|
||||
{
|
||||
openTelemetry.WithMetrics(metrics =>
|
||||
{
|
||||
metrics
|
||||
.AddMeter(IngestionTelemetry.MeterName)
|
||||
.AddAspNetCoreInstrumentation()
|
||||
.AddHttpClientInstrumentation()
|
||||
.AddRuntimeInstrumentation();
|
||||
|
||||
ConfigureExporters(telemetryOptions, metrics);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private static void ConfigureExporters(ExcititorTelemetryOptions options, TracerProviderBuilder tracing)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(options.OtlpEndpoint))
|
||||
{
|
||||
tracing.AddOtlpExporter(otlp =>
|
||||
{
|
||||
otlp.Endpoint = new Uri(options.OtlpEndpoint, UriKind.Absolute);
|
||||
var headers = BuildHeaders(options.OtlpHeaders);
|
||||
if (!string.IsNullOrEmpty(headers))
|
||||
{
|
||||
otlp.Headers = headers;
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (options.ExportConsole)
|
||||
{
|
||||
tracing.AddConsoleExporter();
|
||||
}
|
||||
}
|
||||
|
||||
private static void ConfigureExporters(ExcititorTelemetryOptions options, MeterProviderBuilder metrics)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(options.OtlpEndpoint))
|
||||
{
|
||||
metrics.AddOtlpExporter(otlp =>
|
||||
{
|
||||
otlp.Endpoint = new Uri(options.OtlpEndpoint, UriKind.Absolute);
|
||||
var headers = BuildHeaders(options.OtlpHeaders);
|
||||
if (!string.IsNullOrEmpty(headers))
|
||||
{
|
||||
otlp.Headers = headers;
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (options.ExportConsole)
|
||||
{
|
||||
metrics.AddConsoleExporter();
|
||||
}
|
||||
}
|
||||
|
||||
private static string? BuildHeaders(IReadOnlyDictionary<string, string> headers)
|
||||
{
|
||||
if (headers.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var parts = new List<string>(headers.Count);
|
||||
foreach (var header in headers)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(header.Key) || string.IsNullOrWhiteSpace(header.Value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
parts.Add($"{header.Key}={header.Value}");
|
||||
}
|
||||
|
||||
return parts.Count == 0 ? null : string.Join(',', parts);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Extensions;
|
||||
|
||||
internal static class VexRawRequestMapper
|
||||
{
|
||||
public static VexRawDocument Map(VexIngestRequest request, string tenant, TimeProvider timeProvider)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(request.ProviderId);
|
||||
ArgumentNullException.ThrowIfNull(timeProvider);
|
||||
|
||||
var source = request.Source ?? throw new ArgumentException("source section is required.", nameof(request));
|
||||
var upstream = request.Upstream ?? throw new ArgumentException("upstream section is required.", nameof(request));
|
||||
var content = request.Content ?? throw new ArgumentException("content section is required.", nameof(request));
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(upstream.SourceUri);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(upstream.UpstreamId);
|
||||
|
||||
var providerId = request.ProviderId.Trim();
|
||||
var format = ParseFormat(content.Format);
|
||||
var sourceUri = new Uri(upstream.SourceUri!, UriKind.Absolute);
|
||||
var retrievedAt = upstream.RetrievedAt ?? timeProvider.GetUtcNow();
|
||||
var payload = SerializeContent(content.Raw);
|
||||
var digest = NormalizeDigest(upstream.ContentHash, payload.Span);
|
||||
|
||||
var metadataBuilder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
|
||||
CopyMetadata(metadataBuilder, request.Metadata);
|
||||
CopyMetadata(metadataBuilder, upstream.Provenance);
|
||||
|
||||
metadataBuilder["tenant"] = tenant.Trim().ToLowerInvariant();
|
||||
SetIfMissing(metadataBuilder, "source.vendor", source.Vendor);
|
||||
SetIfMissing(metadataBuilder, "source.connector", source.Connector);
|
||||
SetIfMissing(metadataBuilder, "source.connector_version", source.Version);
|
||||
SetIfMissing(metadataBuilder, "source.stream", source.Stream ?? format.ToString().ToLowerInvariant());
|
||||
SetIfMissing(metadataBuilder, "upstream.id", upstream.UpstreamId);
|
||||
SetIfMissing(metadataBuilder, "upstream.version", upstream.DocumentVersion ?? retrievedAt.ToString("O"));
|
||||
SetIfMissing(metadataBuilder, "content.spec_version", content.SpecVersion);
|
||||
SetIfMissing(metadataBuilder, "content.encoding", content.Encoding);
|
||||
|
||||
var signature = upstream.Signature;
|
||||
metadataBuilder["signature.present"] = (signature?.Present ?? false).ToString();
|
||||
SetIfMissing(metadataBuilder, "signature.format", signature?.Format);
|
||||
SetIfMissing(metadataBuilder, "signature.key_id", signature?.KeyId);
|
||||
SetIfMissing(metadataBuilder, "signature.sig", signature?.Signature);
|
||||
SetIfMissing(metadataBuilder, "signature.certificate", signature?.Certificate);
|
||||
SetIfMissing(metadataBuilder, "signature.digest", signature?.Digest);
|
||||
|
||||
var metadata = metadataBuilder.ToImmutable();
|
||||
|
||||
return new VexRawDocument(
|
||||
providerId,
|
||||
format,
|
||||
sourceUri,
|
||||
retrievedAt,
|
||||
digest,
|
||||
payload,
|
||||
metadata);
|
||||
}
|
||||
|
||||
private static ReadOnlyMemory<byte> SerializeContent(JsonElement element)
|
||||
{
|
||||
var buffer = new ArrayBufferWriter<byte>();
|
||||
using (var writer = new Utf8JsonWriter(buffer, new JsonWriterOptions { Indented = false }))
|
||||
{
|
||||
if (element.ValueKind == JsonValueKind.Undefined)
|
||||
{
|
||||
writer.WriteStartObject();
|
||||
writer.WriteEndObject();
|
||||
}
|
||||
else
|
||||
{
|
||||
element.WriteTo(writer);
|
||||
}
|
||||
}
|
||||
|
||||
return buffer.WrittenMemory.ToArray();
|
||||
}
|
||||
|
||||
private static VexDocumentFormat ParseFormat(string format)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(format))
|
||||
{
|
||||
throw new ArgumentException("content.format is required.", nameof(format));
|
||||
}
|
||||
|
||||
if (Enum.TryParse<VexDocumentFormat>(format, ignoreCase: true, out var parsed))
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
|
||||
throw new ArgumentException($"Unsupported VEX document format {format}.", nameof(format));
|
||||
}
|
||||
|
||||
private static string NormalizeDigest(string? existingDigest, ReadOnlySpan<byte> payload)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(existingDigest))
|
||||
{
|
||||
return existingDigest.Trim();
|
||||
}
|
||||
|
||||
Span<byte> buffer = stackalloc byte[32];
|
||||
if (SHA256.TryHashData(payload, buffer, out _))
|
||||
{
|
||||
return "sha256:" + Convert.ToHexString(buffer).ToLowerInvariant();
|
||||
}
|
||||
|
||||
using var sha = SHA256.Create();
|
||||
var hash = sha.ComputeHash(payload.ToArray());
|
||||
return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static void CopyMetadata(ImmutableDictionary<string, string>.Builder builder, IReadOnlyDictionary<string, string>? values)
|
||||
{
|
||||
if (values is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var kvp in values)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(kvp.Key))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
builder[kvp.Key.Trim()] = kvp.Value?.Trim() ?? string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
private static void SetIfMissing(ImmutableDictionary<string, string>.Builder builder, string key, string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!builder.ContainsKey(key))
|
||||
{
|
||||
builder[key] = value.Trim();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Options;
|
||||
|
||||
internal sealed class ExcititorObservabilityOptions
|
||||
{
|
||||
public TimeSpan IngestWarningThreshold { get; set; } = TimeSpan.FromHours(6);
|
||||
|
||||
public TimeSpan IngestCriticalThreshold { get; set; } = TimeSpan.FromHours(24);
|
||||
|
||||
public TimeSpan LinkWarningThreshold { get; set; } = TimeSpan.FromMinutes(15);
|
||||
|
||||
public TimeSpan LinkCriticalThreshold { get; set; } = TimeSpan.FromHours(1);
|
||||
|
||||
public TimeSpan SignatureWindow { get; set; } = TimeSpan.FromHours(12);
|
||||
|
||||
public double SignatureHealthyCoverage { get; set; } = 0.8;
|
||||
|
||||
public double SignatureWarningCoverage { get; set; } = 0.5;
|
||||
|
||||
public TimeSpan ConflictTrendWindow { get; set; } = TimeSpan.FromHours(24);
|
||||
|
||||
public int ConflictTrendBucketMinutes { get; set; } = 60;
|
||||
|
||||
public double ConflictWarningRatio { get; set; } = 0.15;
|
||||
|
||||
public double ConflictCriticalRatio { get; set; } = 0.3;
|
||||
|
||||
public int MaxConnectorDetails { get; set; } = 50;
|
||||
|
||||
internal TimeSpan GetPositive(TimeSpan candidate, TimeSpan fallback)
|
||||
=> candidate <= TimeSpan.Zero ? fallback : candidate;
|
||||
|
||||
internal double ClampRatio(double value, double fallback)
|
||||
{
|
||||
if (double.IsNaN(value) || double.IsInfinity(value))
|
||||
{
|
||||
return fallback;
|
||||
}
|
||||
|
||||
if (value < 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (value > 1)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Options;
|
||||
|
||||
internal sealed class ExcititorTelemetryOptions
|
||||
{
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
public bool EnableTracing { get; set; } = true;
|
||||
|
||||
public bool EnableMetrics { get; set; } = true;
|
||||
|
||||
public bool ExportConsole { get; set; }
|
||||
|
||||
public string? ServiceName { get; set; }
|
||||
|
||||
public string? OtlpEndpoint { get; set; }
|
||||
|
||||
public Dictionary<string, string> OtlpHeaders { get; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public Dictionary<string, string> ResourceAttributes { get; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
130
src/Excititor/StellaOps.Excititor.WebService/Program.Helpers.cs
Normal file
130
src/Excititor/StellaOps.Excititor.WebService/Program.Helpers.cs
Normal file
@@ -0,0 +1,130 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Excititor.Core.Aoc;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
public partial class Program
|
||||
{
|
||||
private const string TenantHeaderName = "X-Stella-Tenant";
|
||||
|
||||
private static bool TryResolveTenant(HttpContext context, VexMongoStorageOptions options, bool requireHeader, out string tenant, out IResult? problem)
|
||||
{
|
||||
tenant = options.DefaultTenant;
|
||||
problem = null;
|
||||
|
||||
if (context.Request.Headers.TryGetValue(TenantHeaderName, out var headerValues) && headerValues.Count > 0)
|
||||
{
|
||||
var requestedTenant = headerValues[0]?.Trim();
|
||||
if (string.IsNullOrEmpty(requestedTenant))
|
||||
{
|
||||
problem = Results.Problem(detail: "X-Stella-Tenant header must not be empty.", statusCode: StatusCodes.Status400BadRequest, title: "Validation error");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!string.Equals(requestedTenant, options.DefaultTenant, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var detail = string.Format(CultureInfo.InvariantCulture, "Tenant '{0}' is not allowed for this Excititor deployment.", requestedTenant);
|
||||
problem = Results.Problem(detail: detail, statusCode: StatusCodes.Status403Forbidden, title: "Forbidden");
|
||||
return false;
|
||||
}
|
||||
|
||||
tenant = requestedTenant;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (requireHeader)
|
||||
{
|
||||
var detail = string.Format(CultureInfo.InvariantCulture, "{0} header is required.", TenantHeaderName);
|
||||
problem = Results.Problem(detail: detail, statusCode: StatusCodes.Status400BadRequest, title: "Validation error");
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, string> ReadMetadata(BsonValue value)
|
||||
{
|
||||
if (value is not BsonDocument doc || doc.ElementCount == 0)
|
||||
{
|
||||
return new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
var result = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
foreach (var element in doc.Elements)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(element.Name))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
result[element.Name] = element.Value?.ToString() ?? string.Empty;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static bool TryDecodeCursor(string? cursor, out DateTimeOffset timestamp, out string digest)
|
||||
{
|
||||
timestamp = default;
|
||||
digest = string.Empty;
|
||||
if (string.IsNullOrWhiteSpace(cursor))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var payload = Encoding.UTF8.GetString(Convert.FromBase64String(cursor));
|
||||
var parts = payload.Split('|');
|
||||
if (parts.Length != 2)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!DateTimeOffset.TryParse(parts[0], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out timestamp))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
digest = parts[1];
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static string EncodeCursor(DateTime timestamp, string digest)
|
||||
{
|
||||
var payload = string.Format(CultureInfo.InvariantCulture, "{0:O}|{1}", timestamp, digest);
|
||||
return Convert.ToBase64String(Encoding.UTF8.GetBytes(payload));
|
||||
}
|
||||
|
||||
private static IResult ValidationProblem(string message)
|
||||
=> Results.Problem(detail: message, statusCode: StatusCodes.Status400BadRequest, title: "Validation error");
|
||||
|
||||
private static IResult MapGuardException(ExcititorAocGuardException exception)
|
||||
{
|
||||
var violations = exception.Violations.Select(violation => new
|
||||
{
|
||||
code = violation.ErrorCode,
|
||||
path = violation.Path,
|
||||
message = violation.Message
|
||||
});
|
||||
|
||||
return Results.Problem(
|
||||
detail: "VEX document failed Aggregation-Only Contract validation.",
|
||||
statusCode: StatusCodes.Status400BadRequest,
|
||||
title: "AOC violation",
|
||||
extensions: new Dictionary<string, object?>
|
||||
{
|
||||
["violations"] = violations.ToArray(),
|
||||
["primaryCode"] = exception.PrimaryErrorCode,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,10 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using Microsoft.AspNetCore.Authentication;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Attestation.Verification;
|
||||
using StellaOps.Excititor.Attestation.Extensions;
|
||||
@@ -17,13 +20,17 @@ using StellaOps.Excititor.Formats.OpenVEX;
|
||||
using StellaOps.Excititor.Policy;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.WebService.Endpoints;
|
||||
using StellaOps.Excititor.WebService.Extensions;
|
||||
using StellaOps.Excititor.WebService.Options;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
using StellaOps.Excititor.Core.Aoc;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using MongoDB.Driver;
|
||||
using MongoDB.Bson;
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
var configuration = builder.Configuration;
|
||||
var services = builder.Services;
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
var configuration = builder.Configuration;
|
||||
var services = builder.Services;
|
||||
services.AddOptions<VexMongoStorageOptions>()
|
||||
.Bind(configuration.GetSection("Excititor:Storage:Mongo"))
|
||||
.ValidateOnStart();
|
||||
@@ -34,8 +41,11 @@ services.AddCycloneDxNormalizer();
|
||||
services.AddOpenVexNormalizer();
|
||||
services.AddSingleton<IVexSignatureVerifier, NoopVexSignatureVerifier>();
|
||||
services.AddScoped<IVexIngestOrchestrator, VexIngestOrchestrator>();
|
||||
services.AddOptions<ExcititorObservabilityOptions>()
|
||||
.Bind(configuration.GetSection("Excititor:Observability"));
|
||||
services.AddScoped<ExcititorHealthService>();
|
||||
services.AddExcititorAocGuards();
|
||||
services.AddVexExportEngine();
|
||||
services.AddVexExportEngine();
|
||||
services.AddVexExportCacheServices();
|
||||
services.AddVexAttestation();
|
||||
services.Configure<VexAttestationClientOptions>(configuration.GetSection("Excititor:Attestation:Client"));
|
||||
@@ -85,14 +95,17 @@ if (offlineSection.Exists())
|
||||
services.AddEndpointsApiExplorer();
|
||||
services.AddHealthChecks();
|
||||
services.AddSingleton(TimeProvider.System);
|
||||
services.AddMemoryCache();
|
||||
services.AddAuthentication();
|
||||
services.AddAuthorization();
|
||||
services.AddMemoryCache();
|
||||
services.AddAuthentication();
|
||||
services.AddAuthorization();
|
||||
|
||||
builder.ConfigureExcititorTelemetry();
|
||||
|
||||
var app = builder.Build();
|
||||
|
||||
var app = builder.Build();
|
||||
|
||||
app.UseAuthentication();
|
||||
app.UseAuthorization();
|
||||
app.UseAuthentication();
|
||||
app.UseAuthorization();
|
||||
app.UseObservabilityHeaders();
|
||||
|
||||
app.MapGet("/excititor/status", async (HttpContext context,
|
||||
IEnumerable<IVexArtifactStore> artifactStores,
|
||||
@@ -143,26 +156,428 @@ app.MapGet("/excititor/statements/{vulnerabilityId}/{productKey}", async (
|
||||
return Results.Ok(claims);
|
||||
});
|
||||
|
||||
app.MapPost("/excititor/admin/backfill-statements", async (
|
||||
VexStatementBackfillRequest? request,
|
||||
VexStatementBackfillService backfillService,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
app.MapPost("/excititor/admin/backfill-statements", async (
|
||||
VexStatementBackfillRequest? request,
|
||||
VexStatementBackfillService backfillService,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
request ??= new VexStatementBackfillRequest();
|
||||
var result = await backfillService.RunAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
var message = FormattableString.Invariant(
|
||||
$"Backfill completed: evaluated {result.DocumentsEvaluated}, backfilled {result.DocumentsBackfilled}, claims written {result.ClaimsWritten}, skipped {result.SkippedExisting}, failures {result.NormalizationFailures}.");
|
||||
|
||||
return Results.Ok(new
|
||||
{
|
||||
message,
|
||||
summary = result
|
||||
});
|
||||
});
|
||||
|
||||
IngestEndpoints.MapIngestEndpoints(app);
|
||||
ResolveEndpoint.MapResolveEndpoint(app);
|
||||
MirrorEndpoints.MapMirrorEndpoints(app);
|
||||
return Results.Ok(new
|
||||
{
|
||||
message,
|
||||
summary = result
|
||||
});
|
||||
});
|
||||
|
||||
app.MapPost("/ingest/vex", async (
|
||||
HttpContext context,
|
||||
VexIngestRequest request,
|
||||
IVexRawStore rawStore,
|
||||
IOptions<VexMongoStorageOptions> storageOptions,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<Program> logger,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.admin");
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: true, out var tenant, out var tenantError))
|
||||
{
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
VexRawDocument document;
|
||||
try
|
||||
{
|
||||
document = VexRawRequestMapper.Map(request, tenant, timeProvider);
|
||||
}
|
||||
catch (Exception ex) when (ex is ArgumentException or InvalidOperationException or FormatException)
|
||||
{
|
||||
return ValidationProblem(ex.Message);
|
||||
}
|
||||
|
||||
var existing = await rawStore.FindByDigestAsync(document.Digest, cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
await rawStore.StoreAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (ExcititorAocGuardException guardException)
|
||||
{
|
||||
logger.LogWarning(
|
||||
guardException,
|
||||
"AOC guard rejected VEX ingest tenant={Tenant} digest={Digest}",
|
||||
tenant,
|
||||
document.Digest);
|
||||
return MapGuardException(guardException);
|
||||
}
|
||||
|
||||
var inserted = existing is null;
|
||||
if (inserted)
|
||||
{
|
||||
context.Response.Headers.Location = $"/vex/raw/{Uri.EscapeDataString(document.Digest)}";
|
||||
}
|
||||
|
||||
var response = new VexIngestResponse(document.Digest, inserted, tenant, document.RetrievedAt);
|
||||
return Results.Json(response, statusCode: inserted ? StatusCodes.Status201Created : StatusCodes.Status200OK);
|
||||
});
|
||||
|
||||
app.MapGet("/vex/raw", async (
|
||||
HttpContext context,
|
||||
IMongoDatabase database,
|
||||
IOptions<VexMongoStorageOptions> storageOptions,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out _, out var tenantError))
|
||||
{
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
|
||||
var query = context.Request.Query;
|
||||
var filters = new List<FilterDefinition<BsonDocument>>();
|
||||
var builder = Builders<BsonDocument>.Filter;
|
||||
|
||||
if (query.TryGetValue("providerId", out var providerValues))
|
||||
{
|
||||
var providers = providerValues
|
||||
.Where(static value => !string.IsNullOrWhiteSpace(value))
|
||||
.Select(static value => value!.Trim())
|
||||
.ToArray();
|
||||
if (providers.Length > 0)
|
||||
{
|
||||
filters.Add(builder.In("ProviderId", providers));
|
||||
}
|
||||
}
|
||||
|
||||
if (query.TryGetValue("digest", out var digestValues))
|
||||
{
|
||||
var digests = digestValues
|
||||
.Where(static value => !string.IsNullOrWhiteSpace(value))
|
||||
.Select(static value => value!.Trim())
|
||||
.ToArray();
|
||||
if (digests.Length > 0)
|
||||
{
|
||||
filters.Add(builder.In("Digest", digests));
|
||||
}
|
||||
}
|
||||
|
||||
if (query.TryGetValue("format", out var formatValues))
|
||||
{
|
||||
var formats = formatValues
|
||||
.Where(static value => !string.IsNullOrWhiteSpace(value))
|
||||
.Select(static value => value!.Trim().ToLowerInvariant())
|
||||
.ToArray();
|
||||
if (formats.Length > 0)
|
||||
{
|
||||
filters.Add(builder.In("Format", formats));
|
||||
}
|
||||
}
|
||||
|
||||
if (query.TryGetValue("since", out var sinceValues) && DateTimeOffset.TryParse(sinceValues.FirstOrDefault(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var sinceValue))
|
||||
{
|
||||
filters.Add(builder.Gte("RetrievedAt", sinceValue.UtcDateTime));
|
||||
}
|
||||
|
||||
var cursorToken = query.TryGetValue("cursor", out var cursorValues) ? cursorValues.FirstOrDefault() : null;
|
||||
DateTime? cursorTimestamp = null;
|
||||
string? cursorDigest = null;
|
||||
if (!string.IsNullOrWhiteSpace(cursorToken) && TryDecodeCursor(cursorToken, out var cursorTime, out var cursorId))
|
||||
{
|
||||
cursorTimestamp = cursorTime.UtcDateTime;
|
||||
cursorDigest = cursorId;
|
||||
}
|
||||
|
||||
if (cursorTimestamp is not null && cursorDigest is not null)
|
||||
{
|
||||
var ltTime = builder.Lt("RetrievedAt", cursorTimestamp.Value);
|
||||
var eqTimeLtDigest = builder.And(
|
||||
builder.Eq("RetrievedAt", cursorTimestamp.Value),
|
||||
builder.Lt("Digest", cursorDigest));
|
||||
filters.Add(builder.Or(ltTime, eqTimeLtDigest));
|
||||
}
|
||||
|
||||
var limit = 50;
|
||||
if (query.TryGetValue("limit", out var limitValues) && int.TryParse(limitValues.FirstOrDefault(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var requestedLimit))
|
||||
{
|
||||
limit = Math.Clamp(requestedLimit, 1, 200);
|
||||
}
|
||||
|
||||
var filter = filters.Count == 0 ? builder.Empty : builder.And(filters);
|
||||
var sort = Builders<BsonDocument>.Sort.Descending("RetrievedAt").Descending("Digest");
|
||||
var documents = await collection
|
||||
.Find(filter)
|
||||
.Sort(sort)
|
||||
.Limit(limit)
|
||||
.Project(Builders<BsonDocument>.Projection.Include("Digest").Include("ProviderId").Include("Format").Include("SourceUri").Include("RetrievedAt").Include("Metadata").Include("GridFsObjectId"))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var summaries = new List<VexRawSummaryResponse>(documents.Count);
|
||||
foreach (var document in documents)
|
||||
{
|
||||
var digest = document.TryGetValue("Digest", out var digestValue) && digestValue.IsString ? digestValue.AsString : string.Empty;
|
||||
var providerId = document.TryGetValue("ProviderId", out var providerValue) && providerValue.IsString ? providerValue.AsString : string.Empty;
|
||||
var format = document.TryGetValue("Format", out var formatValue) && formatValue.IsString ? formatValue.AsString : string.Empty;
|
||||
var sourceUri = document.TryGetValue("SourceUri", out var sourceValue) && sourceValue.IsString ? sourceValue.AsString : string.Empty;
|
||||
var retrievedAt = document.TryGetValue("RetrievedAt", out var retrievedValue) && retrievedValue is BsonDateTime bsonDate
|
||||
? bsonDate.ToUniversalTime()
|
||||
: DateTime.UtcNow;
|
||||
var metadata = ReadMetadata(document.TryGetValue("Metadata", out var metadataValue) ? metadataValue : BsonNull.Value);
|
||||
var inlineContent = !document.TryGetValue("GridFsObjectId", out var gridId) || gridId.IsBsonNull || (gridId.IsString && string.IsNullOrWhiteSpace(gridId.AsString));
|
||||
|
||||
summaries.Add(new VexRawSummaryResponse(
|
||||
digest,
|
||||
providerId,
|
||||
format,
|
||||
sourceUri,
|
||||
new DateTimeOffset(retrievedAt),
|
||||
inlineContent,
|
||||
metadata));
|
||||
}
|
||||
|
||||
var hasMore = documents.Count == limit;
|
||||
string? nextCursor = null;
|
||||
if (hasMore && documents.Count > 0)
|
||||
{
|
||||
var last = documents[^1];
|
||||
var lastTime = last.GetValue("RetrievedAt", BsonNull.Value).ToUniversalTime();
|
||||
var lastDigest = last.GetValue("Digest", BsonNull.Value).AsString;
|
||||
nextCursor = EncodeCursor(lastTime, lastDigest);
|
||||
}
|
||||
|
||||
return Results.Json(new VexRawListResponse(summaries, nextCursor, hasMore));
|
||||
});
|
||||
|
||||
app.MapGet("/vex/raw/{digest}", async (
|
||||
string digest,
|
||||
HttpContext context,
|
||||
IVexRawStore rawStore,
|
||||
IOptions<VexMongoStorageOptions> storageOptions,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out _, out var tenantError))
|
||||
{
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return ValidationProblem("digest is required.");
|
||||
}
|
||||
|
||||
var record = await rawStore.FindByDigestAsync(digest.Trim(), cancellationToken).ConfigureAwait(false);
|
||||
if (record is null)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
var rawDocument = VexRawDocumentMapper.ToRawModel(record, storageOptions.Value.DefaultTenant);
|
||||
var response = new VexRawRecordResponse(record.Digest, rawDocument, record.RetrievedAt);
|
||||
return Results.Json(response);
|
||||
});
|
||||
|
||||
app.MapGet("/vex/raw/{digest}/provenance", async (
|
||||
string digest,
|
||||
HttpContext context,
|
||||
IVexRawStore rawStore,
|
||||
IOptions<VexMongoStorageOptions> storageOptions,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out _, out var tenantError))
|
||||
{
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return ValidationProblem("digest is required.");
|
||||
}
|
||||
|
||||
var record = await rawStore.FindByDigestAsync(digest.Trim(), cancellationToken).ConfigureAwait(false);
|
||||
if (record is null)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
var rawDocument = VexRawDocumentMapper.ToRawModel(record, storageOptions.Value.DefaultTenant);
|
||||
var response = new VexRawProvenanceResponse(
|
||||
record.Digest,
|
||||
rawDocument.Tenant,
|
||||
rawDocument.Source,
|
||||
rawDocument.Upstream,
|
||||
record.RetrievedAt);
|
||||
return Results.Json(response);
|
||||
});
|
||||
|
||||
app.MapPost("/aoc/verify", async (
|
||||
HttpContext context,
|
||||
VexAocVerifyRequest? request,
|
||||
IMongoDatabase database,
|
||||
IVexRawStore rawStore,
|
||||
IVexRawWriteGuard guard,
|
||||
IOptions<VexMongoStorageOptions> storageOptions,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.admin");
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out var tenant, out var tenantError))
|
||||
{
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
var now = timeProvider.GetUtcNow();
|
||||
var since = (request?.Since ?? now.AddHours(-24)).UtcDateTime;
|
||||
var until = (request?.Until ?? now).UtcDateTime;
|
||||
if (since >= until)
|
||||
{
|
||||
since = until.AddHours(-1);
|
||||
}
|
||||
|
||||
var limit = Math.Clamp(request?.Limit ?? 100, 1, 500);
|
||||
var sources = request?.Sources?
|
||||
.Where(static value => !string.IsNullOrWhiteSpace(value))
|
||||
.Select(static value => value!.Trim())
|
||||
.ToArray();
|
||||
var requestedCodes = request?.Codes?
|
||||
.Where(static value => !string.IsNullOrWhiteSpace(value))
|
||||
.Select(static value => value!.Trim())
|
||||
.ToArray();
|
||||
|
||||
var builder = Builders<BsonDocument>.Filter;
|
||||
var filter = builder.And(
|
||||
builder.Gte("RetrievedAt", since),
|
||||
builder.Lte("RetrievedAt", until));
|
||||
|
||||
if (sources is { Length: > 0 })
|
||||
{
|
||||
filter &= builder.In("ProviderId", sources);
|
||||
}
|
||||
|
||||
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
|
||||
var digests = await collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<BsonDocument>.Sort.Descending("RetrievedAt"))
|
||||
.Limit(limit)
|
||||
.Project(Builders<BsonDocument>.Projection.Include("Digest").Include("RetrievedAt").Include("ProviderId"))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var checkedCount = 0;
|
||||
var violationMap = new Dictionary<string, (int Count, List<VexAocVerifyViolationExample> Examples)>(StringComparer.OrdinalIgnoreCase);
|
||||
const int MaxExamplesPerCode = 5;
|
||||
|
||||
foreach (var digestDocument in digests)
|
||||
{
|
||||
var digestValue = digestDocument.GetValue("Digest", BsonNull.Value).AsString;
|
||||
var provider = digestDocument.GetValue("ProviderId", BsonNull.Value).AsString;
|
||||
|
||||
var domainDocument = await rawStore.FindByDigestAsync(digestValue, cancellationToken).ConfigureAwait(false);
|
||||
if (domainDocument is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var rawDocument = VexRawDocumentMapper.ToRawModel(domainDocument, storageOptions.Value.DefaultTenant);
|
||||
try
|
||||
{
|
||||
guard.EnsureValid(rawDocument);
|
||||
checkedCount++;
|
||||
}
|
||||
catch (ExcititorAocGuardException guardException)
|
||||
{
|
||||
checkedCount++;
|
||||
foreach (var violation in guardException.Violations)
|
||||
{
|
||||
var code = violation.ErrorCode;
|
||||
if (requestedCodes is { Length: > 0 } && !requestedCodes.Contains(code, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!violationMap.TryGetValue(code, out var aggregate))
|
||||
{
|
||||
aggregate = (0, new List<VexAocVerifyViolationExample>(MaxExamplesPerCode));
|
||||
}
|
||||
|
||||
aggregate.Count++;
|
||||
if (aggregate.Examples.Count < MaxExamplesPerCode)
|
||||
{
|
||||
aggregate.Examples.Add(new VexAocVerifyViolationExample(
|
||||
provider,
|
||||
digestValue,
|
||||
rawDocument.Upstream.ContentHash,
|
||||
violation.Path));
|
||||
}
|
||||
|
||||
violationMap[code] = aggregate;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var violations = violationMap
|
||||
.Select(pair => new VexAocVerifyViolation(pair.Key, pair.Value.Count, pair.Value.Examples))
|
||||
.OrderByDescending(violation => violation.Count)
|
||||
.ToList();
|
||||
|
||||
var response = new VexAocVerifyResponse(
|
||||
tenant,
|
||||
new VexAocVerifyWindow(new DateTimeOffset(since, TimeSpan.Zero), new DateTimeOffset(until, TimeSpan.Zero)),
|
||||
new VexAocVerifyChecked(0, checkedCount),
|
||||
violations,
|
||||
new VexAocVerifyMetrics(checkedCount, violations.Sum(v => v.Count)),
|
||||
digests.Count == limit);
|
||||
|
||||
return Results.Json(response);
|
||||
});
|
||||
|
||||
app.MapGet("/obs/excititor/health", async (
|
||||
HttpContext httpContext,
|
||||
ExcititorHealthService healthService,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(httpContext, "vex.admin");
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
var payload = await healthService.GetAsync(cancellationToken).ConfigureAwait(false);
|
||||
return Results.Ok(payload);
|
||||
});
|
||||
|
||||
IngestEndpoints.MapIngestEndpoints(app);
|
||||
ResolveEndpoint.MapResolveEndpoint(app);
|
||||
MirrorEndpoints.MapMirrorEndpoints(app);
|
||||
|
||||
app.Run();
|
||||
|
||||
|
||||
@@ -0,0 +1,667 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.WebService.Options;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Services;
|
||||
|
||||
internal sealed class ExcititorHealthService
|
||||
{
|
||||
private const string RetrievedAtField = "RetrievedAt";
|
||||
private const string MetadataField = "Metadata";
|
||||
private const string CalculatedAtField = "CalculatedAt";
|
||||
private const string ConflictsField = "Conflicts";
|
||||
private const string ConflictStatusField = "Status";
|
||||
|
||||
private readonly IMongoDatabase _database;
|
||||
private readonly IVexProviderStore _providerStore;
|
||||
private readonly IVexConnectorStateRepository _stateRepository;
|
||||
private readonly IReadOnlyDictionary<string, VexConnectorDescriptor> _connectors;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ExcititorObservabilityOptions _options;
|
||||
private readonly ILogger<ExcititorHealthService> _logger;
|
||||
|
||||
public ExcititorHealthService(
|
||||
IMongoDatabase database,
|
||||
IVexProviderStore providerStore,
|
||||
IVexConnectorStateRepository stateRepository,
|
||||
IEnumerable<IVexConnector> connectors,
|
||||
TimeProvider timeProvider,
|
||||
IOptions<ExcititorObservabilityOptions> options,
|
||||
ILogger<ExcititorHealthService> logger)
|
||||
{
|
||||
_database = database ?? throw new ArgumentNullException(nameof(database));
|
||||
_providerStore = providerStore ?? throw new ArgumentNullException(nameof(providerStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_options = options?.Value ?? new ExcititorObservabilityOptions();
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
|
||||
if (connectors is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(connectors));
|
||||
}
|
||||
|
||||
_connectors = connectors
|
||||
.GroupBy(connector => connector.Id, StringComparer.OrdinalIgnoreCase)
|
||||
.ToDictionary(
|
||||
group => group.Key,
|
||||
group => DescribeConnector(group.First()),
|
||||
StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public async Task<ExcititorHealthDocument> GetAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
var providersTask = _providerStore.ListAsync(cancellationToken).AsTask();
|
||||
var statesTask = _stateRepository.ListAsync(cancellationToken).AsTask();
|
||||
var signatureTask = LoadSignatureSnapshotAsync(now, cancellationToken);
|
||||
var conflictTask = LoadConflictSnapshotAsync(now, cancellationToken);
|
||||
var linkTask = LoadLinkSnapshotAsync(cancellationToken);
|
||||
|
||||
await Task.WhenAll(providersTask, statesTask, signatureTask, conflictTask, linkTask).ConfigureAwait(false);
|
||||
|
||||
var ingest = BuildIngestSection(now, providersTask.Result, statesTask.Result);
|
||||
var link = BuildLinkSection(now, linkTask.Result);
|
||||
var conflicts = BuildConflictSection(conflictTask.Result, link);
|
||||
var signature = BuildSignatureSection(signatureTask.Result);
|
||||
|
||||
return new ExcititorHealthDocument(
|
||||
now,
|
||||
ingest,
|
||||
link,
|
||||
signature,
|
||||
conflicts);
|
||||
}
|
||||
|
||||
private IngestHealthSection BuildIngestSection(
|
||||
DateTimeOffset now,
|
||||
IReadOnlyCollection<VexProvider> providers,
|
||||
IReadOnlyCollection<VexConnectorState> states)
|
||||
{
|
||||
var providerNames = providers
|
||||
.GroupBy(provider => provider.Id, StringComparer.OrdinalIgnoreCase)
|
||||
.ToDictionary(
|
||||
group => group.Key,
|
||||
group => group.First().DisplayName,
|
||||
StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var stateMap = states
|
||||
.ToDictionary(state => state.ConnectorId, state => state, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var warningThreshold = _options.GetPositive(_options.IngestWarningThreshold, TimeSpan.FromHours(6));
|
||||
var criticalThreshold = _options.GetPositive(_options.IngestCriticalThreshold, TimeSpan.FromHours(24));
|
||||
|
||||
var connectorHealth = new List<ConnectorHealth>(_connectors.Count);
|
||||
foreach (var descriptor in _connectors.Values.OrderBy(d => d.Id, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
stateMap.TryGetValue(descriptor.Id, out var state);
|
||||
var displayName = providerNames.TryGetValue(descriptor.Id, out var name)
|
||||
? name
|
||||
: descriptor.DisplayName;
|
||||
|
||||
var lastSuccess = state?.LastSuccessAt ?? state?.LastUpdated;
|
||||
double? lagSeconds = null;
|
||||
if (lastSuccess is not null)
|
||||
{
|
||||
var lag = now - lastSuccess.Value;
|
||||
if (lag < TimeSpan.Zero)
|
||||
{
|
||||
lag = TimeSpan.Zero;
|
||||
}
|
||||
|
||||
lagSeconds = lag.TotalSeconds;
|
||||
}
|
||||
|
||||
var status = DetermineIngestStatus(
|
||||
state?.FailureCount ?? 0,
|
||||
lagSeconds,
|
||||
warningThreshold.TotalSeconds,
|
||||
criticalThreshold.TotalSeconds);
|
||||
|
||||
connectorHealth.Add(new ConnectorHealth(
|
||||
descriptor.Id,
|
||||
displayName,
|
||||
status,
|
||||
lastSuccess,
|
||||
state?.LastUpdated,
|
||||
state?.NextEligibleRun,
|
||||
lagSeconds,
|
||||
state?.FailureCount ?? 0,
|
||||
state?.LastFailureReason));
|
||||
}
|
||||
|
||||
var overallStatus = ReduceStatuses(connectorHealth.Select(ch => ch.Status));
|
||||
double? maxLag = connectorHealth.Any(ch => ch.LagSeconds.HasValue)
|
||||
? connectorHealth.Max(ch => ch.LagSeconds ?? 0d)
|
||||
: null;
|
||||
|
||||
var maxDetails = _options.MaxConnectorDetails <= 0 ? 50 : _options.MaxConnectorDetails;
|
||||
var projected = connectorHealth
|
||||
.OrderByDescending(ch => SeverityRank(ch.Status))
|
||||
.ThenByDescending(ch => ch.LagSeconds ?? -1)
|
||||
.ThenBy(ch => ch.ConnectorId, StringComparer.OrdinalIgnoreCase)
|
||||
.Take(maxDetails)
|
||||
.ToList();
|
||||
|
||||
return new IngestHealthSection(overallStatus, maxLag, projected);
|
||||
}
|
||||
|
||||
private LinkHealthSection BuildLinkSection(DateTimeOffset now, LinkSnapshot snapshot)
|
||||
{
|
||||
TimeSpan? lag = null;
|
||||
if (snapshot.LastConsensusAt is { } calculatedAt)
|
||||
{
|
||||
lag = now - calculatedAt;
|
||||
if (lag < TimeSpan.Zero)
|
||||
{
|
||||
lag = TimeSpan.Zero;
|
||||
}
|
||||
}
|
||||
|
||||
var warning = _options.GetPositive(_options.LinkWarningThreshold, TimeSpan.FromMinutes(15));
|
||||
var critical = _options.GetPositive(_options.LinkCriticalThreshold, TimeSpan.FromHours(1));
|
||||
|
||||
var status = DetermineLagStatus(lag, warning, critical);
|
||||
|
||||
return new LinkHealthSection(
|
||||
status,
|
||||
snapshot.LastConsensusAt,
|
||||
lag?.TotalSeconds,
|
||||
snapshot.TotalDocuments,
|
||||
snapshot.DocumentsWithConflicts);
|
||||
}
|
||||
|
||||
private ConflictHealthSection BuildConflictSection(ConflictSnapshot snapshot, LinkHealthSection link)
|
||||
{
|
||||
var warningRatio = _options.ClampRatio(_options.ConflictWarningRatio, 0.15);
|
||||
var criticalRatio = _options.ClampRatio(_options.ConflictCriticalRatio, 0.3);
|
||||
|
||||
string status;
|
||||
if (link.TotalDocuments <= 0)
|
||||
{
|
||||
status = "unknown";
|
||||
}
|
||||
else
|
||||
{
|
||||
var ratio = (double)snapshot.DocumentsWithConflicts / link.TotalDocuments;
|
||||
if (ratio >= criticalRatio)
|
||||
{
|
||||
status = "critical";
|
||||
}
|
||||
else if (ratio >= warningRatio)
|
||||
{
|
||||
status = "warning";
|
||||
}
|
||||
else
|
||||
{
|
||||
status = "healthy";
|
||||
}
|
||||
}
|
||||
|
||||
return new ConflictHealthSection(
|
||||
status,
|
||||
snapshot.WindowStart,
|
||||
snapshot.WindowEnd,
|
||||
snapshot.DocumentsWithConflicts,
|
||||
snapshot.TotalConflicts,
|
||||
snapshot.ByStatus,
|
||||
snapshot.Trend);
|
||||
}
|
||||
|
||||
private SignatureHealthSection BuildSignatureSection(SignatureSnapshot snapshot)
|
||||
{
|
||||
if (snapshot.DocumentsEvaluated == 0)
|
||||
{
|
||||
return new SignatureHealthSection(
|
||||
"unknown",
|
||||
snapshot.WindowStart,
|
||||
snapshot.WindowEnd,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0);
|
||||
}
|
||||
|
||||
var coverage = snapshot.DocumentsEvaluated == 0
|
||||
? 0d
|
||||
: (double)snapshot.Verified / snapshot.DocumentsEvaluated;
|
||||
|
||||
var healthy = _options.ClampRatio(_options.SignatureHealthyCoverage, 0.8);
|
||||
var warning = _options.ClampRatio(_options.SignatureWarningCoverage, 0.5);
|
||||
if (warning > healthy)
|
||||
{
|
||||
warning = healthy;
|
||||
}
|
||||
|
||||
var status = coverage switch
|
||||
{
|
||||
var value when value >= healthy => "healthy",
|
||||
var value when value >= warning => "warning",
|
||||
_ => "critical"
|
||||
};
|
||||
|
||||
var failures = Math.Max(0, snapshot.WithSignatures - snapshot.Verified);
|
||||
var unsigned = Math.Max(0, snapshot.DocumentsEvaluated - snapshot.WithSignatures);
|
||||
|
||||
return new SignatureHealthSection(
|
||||
status,
|
||||
snapshot.WindowStart,
|
||||
snapshot.WindowEnd,
|
||||
snapshot.DocumentsEvaluated,
|
||||
snapshot.WithSignatures,
|
||||
snapshot.Verified,
|
||||
failures,
|
||||
unsigned,
|
||||
coverage);
|
||||
}
|
||||
|
||||
private async Task<SignatureSnapshot> LoadSignatureSnapshotAsync(DateTimeOffset now, CancellationToken cancellationToken)
|
||||
{
|
||||
var window = _options.GetPositive(_options.SignatureWindow, TimeSpan.FromHours(12));
|
||||
var windowStart = now - window;
|
||||
|
||||
var collection = _database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
|
||||
var filter = Builders<BsonDocument>.Filter.Gte(RetrievedAtField, windowStart.UtcDateTime);
|
||||
var projection = Builders<BsonDocument>.Projection
|
||||
.Include(MetadataField)
|
||||
.Include(RetrievedAtField);
|
||||
|
||||
List<BsonDocument> documents;
|
||||
try
|
||||
{
|
||||
documents = await collection
|
||||
.Find(filter)
|
||||
.Project(projection)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to load signature window metrics.");
|
||||
documents = new List<BsonDocument>();
|
||||
}
|
||||
|
||||
var evaluated = 0;
|
||||
var withSignatures = 0;
|
||||
var verified = 0;
|
||||
|
||||
foreach (var document in documents)
|
||||
{
|
||||
evaluated++;
|
||||
if (!document.TryGetValue(MetadataField, out var metadataValue) ||
|
||||
metadataValue is not BsonDocument metadata ||
|
||||
metadata.ElementCount == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (TryGetBoolean(metadata, "signature.present", out var present) && present)
|
||||
{
|
||||
withSignatures++;
|
||||
}
|
||||
|
||||
if (TryGetBoolean(metadata, "signature.verified", out var verifiedFlag) && verifiedFlag)
|
||||
{
|
||||
verified++;
|
||||
}
|
||||
}
|
||||
|
||||
return new SignatureSnapshot(windowStart, now, evaluated, withSignatures, verified);
|
||||
}
|
||||
|
||||
private async Task<LinkSnapshot> LoadLinkSnapshotAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var collection = _database.GetCollection<BsonDocument>(VexMongoCollectionNames.Consensus);
|
||||
|
||||
BsonDocument? latest = null;
|
||||
try
|
||||
{
|
||||
latest = await collection
|
||||
.Find(Builders<BsonDocument>.Filter.Empty)
|
||||
.Sort(Builders<BsonDocument>.Sort.Descending(CalculatedAtField))
|
||||
.Project(Builders<BsonDocument>.Projection.Include(CalculatedAtField))
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to read latest consensus document.");
|
||||
}
|
||||
|
||||
DateTimeOffset? lastConsensusAt = null;
|
||||
if (latest is not null &&
|
||||
latest.TryGetValue(CalculatedAtField, out var dateValue))
|
||||
{
|
||||
var utc = TryReadDateTime(dateValue);
|
||||
if (utc is not null)
|
||||
{
|
||||
lastConsensusAt = new DateTimeOffset(utc.Value, TimeSpan.Zero);
|
||||
}
|
||||
}
|
||||
|
||||
long totalDocuments = 0;
|
||||
long conflictDocuments = 0;
|
||||
|
||||
try
|
||||
{
|
||||
totalDocuments = await collection.EstimatedDocumentCountAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
conflictDocuments = await collection.CountDocumentsAsync(
|
||||
Builders<BsonDocument>.Filter.Exists($"{ConflictsField}.0"),
|
||||
cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to compute consensus counts.");
|
||||
}
|
||||
|
||||
return new LinkSnapshot(lastConsensusAt, totalDocuments, conflictDocuments);
|
||||
}
|
||||
|
||||
private async Task<ConflictSnapshot> LoadConflictSnapshotAsync(DateTimeOffset now, CancellationToken cancellationToken)
|
||||
{
|
||||
var window = _options.GetPositive(_options.ConflictTrendWindow, TimeSpan.FromHours(24));
|
||||
var windowStart = now - window;
|
||||
var collection = _database.GetCollection<BsonDocument>(VexMongoCollectionNames.Consensus);
|
||||
|
||||
var filter = Builders<BsonDocument>.Filter.And(
|
||||
Builders<BsonDocument>.Filter.Gte(CalculatedAtField, windowStart.UtcDateTime),
|
||||
Builders<BsonDocument>.Filter.Exists($"{ConflictsField}.0"));
|
||||
|
||||
var projection = Builders<BsonDocument>.Projection
|
||||
.Include(CalculatedAtField)
|
||||
.Include(ConflictsField);
|
||||
|
||||
List<BsonDocument> documents;
|
||||
try
|
||||
{
|
||||
documents = await collection
|
||||
.Find(filter)
|
||||
.Project(projection)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to load conflict trend window.");
|
||||
documents = new List<BsonDocument>();
|
||||
}
|
||||
|
||||
var byStatus = new Dictionary<string, long>(StringComparer.OrdinalIgnoreCase);
|
||||
var timeline = new SortedDictionary<long, long>();
|
||||
long totalConflicts = 0;
|
||||
long docsWithConflicts = 0;
|
||||
var bucketMinutes = Math.Max(1, _options.ConflictTrendBucketMinutes);
|
||||
var bucketTicks = TimeSpan.FromMinutes(bucketMinutes).Ticks;
|
||||
|
||||
foreach (var doc in documents)
|
||||
{
|
||||
if (!doc.TryGetValue(ConflictsField, out var conflictsValue) ||
|
||||
conflictsValue is not BsonArray conflicts ||
|
||||
conflicts.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
docsWithConflicts++;
|
||||
totalConflicts += conflicts.Count;
|
||||
|
||||
foreach (var conflictValue in conflicts.OfType<BsonDocument>())
|
||||
{
|
||||
var status = conflictValue.TryGetValue(ConflictStatusField, out var statusValue) && statusValue.IsString
|
||||
? statusValue.AsString
|
||||
: "unknown";
|
||||
|
||||
if (string.IsNullOrWhiteSpace(status))
|
||||
{
|
||||
status = "unknown";
|
||||
}
|
||||
|
||||
byStatus[status] = byStatus.TryGetValue(status, out var current)
|
||||
? current + 1
|
||||
: 1;
|
||||
}
|
||||
|
||||
if (doc.TryGetValue(CalculatedAtField, out var calculatedValue))
|
||||
{
|
||||
var utc = TryReadDateTime(calculatedValue);
|
||||
if (utc is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var alignedTicks = AlignTicks(utc.Value, bucketTicks);
|
||||
timeline[alignedTicks] = timeline.TryGetValue(alignedTicks, out var current)
|
||||
? current + conflicts.Count
|
||||
: conflicts.Count;
|
||||
}
|
||||
}
|
||||
|
||||
var trend = timeline
|
||||
.Select(pair => new ConflictTrendPoint(
|
||||
new DateTimeOffset(pair.Key, TimeSpan.Zero),
|
||||
pair.Value))
|
||||
.ToList();
|
||||
|
||||
return new ConflictSnapshot(
|
||||
windowStart,
|
||||
now,
|
||||
docsWithConflicts,
|
||||
totalConflicts,
|
||||
new Dictionary<string, long>(byStatus, StringComparer.OrdinalIgnoreCase),
|
||||
trend);
|
||||
}
|
||||
|
||||
private static string DetermineIngestStatus(int failureCount, double? lagSeconds, double warningSeconds, double criticalSeconds)
|
||||
{
|
||||
if (failureCount > 0)
|
||||
{
|
||||
return "failing";
|
||||
}
|
||||
|
||||
if (lagSeconds is null)
|
||||
{
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
if (lagSeconds.Value >= criticalSeconds)
|
||||
{
|
||||
return "critical";
|
||||
}
|
||||
|
||||
if (lagSeconds.Value >= warningSeconds)
|
||||
{
|
||||
return "warning";
|
||||
}
|
||||
|
||||
return "healthy";
|
||||
}
|
||||
|
||||
private static string DetermineLagStatus(TimeSpan? lag, TimeSpan warning, TimeSpan critical)
|
||||
{
|
||||
if (lag is null)
|
||||
{
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
if (lag.Value >= critical)
|
||||
{
|
||||
return "critical";
|
||||
}
|
||||
|
||||
if (lag.Value >= warning)
|
||||
{
|
||||
return "warning";
|
||||
}
|
||||
|
||||
return "healthy";
|
||||
}
|
||||
|
||||
private static string ReduceStatuses(IEnumerable<string> statuses)
|
||||
{
|
||||
var highest = "unknown";
|
||||
var highestRank = -1;
|
||||
foreach (var status in statuses)
|
||||
{
|
||||
var rank = SeverityRank(status);
|
||||
if (rank > highestRank)
|
||||
{
|
||||
highestRank = rank;
|
||||
highest = status;
|
||||
}
|
||||
}
|
||||
|
||||
return highest;
|
||||
}
|
||||
|
||||
private static int SeverityRank(string? status)
|
||||
=> status?.ToLowerInvariant() switch
|
||||
{
|
||||
"critical" => 4,
|
||||
"failing" => 3,
|
||||
"warning" => 2,
|
||||
"unknown" => 1,
|
||||
_ => 0,
|
||||
};
|
||||
|
||||
private static long AlignTicks(DateTime dateTimeUtc, long bucketTicks)
|
||||
{
|
||||
var ticks = dateTimeUtc.Ticks;
|
||||
return ticks - (ticks % bucketTicks);
|
||||
}
|
||||
|
||||
private static DateTime? TryReadDateTime(BsonValue value)
|
||||
{
|
||||
if (value is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (value.IsBsonDateTime)
|
||||
{
|
||||
return value.AsBsonDateTime.ToUniversalTime();
|
||||
}
|
||||
|
||||
if (value.IsString &&
|
||||
DateTime.TryParse(
|
||||
value.AsString,
|
||||
CultureInfo.InvariantCulture,
|
||||
DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal,
|
||||
out var parsed))
|
||||
{
|
||||
return DateTime.SpecifyKind(parsed, DateTimeKind.Utc);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool TryGetBoolean(BsonDocument document, string key, out bool value)
|
||||
{
|
||||
value = default;
|
||||
if (!document.TryGetValue(key, out var bsonValue))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (bsonValue.IsBoolean)
|
||||
{
|
||||
value = bsonValue.AsBoolean;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (bsonValue.IsString && bool.TryParse(bsonValue.AsString, out var parsed))
|
||||
{
|
||||
value = parsed;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static VexConnectorDescriptor DescribeConnector(IVexConnector connector)
|
||||
=> connector switch
|
||||
{
|
||||
VexConnectorBase baseConnector => baseConnector.Descriptor,
|
||||
_ => new VexConnectorDescriptor(connector.Id, connector.Kind, connector.Id)
|
||||
};
|
||||
|
||||
private sealed record LinkSnapshot(DateTimeOffset? LastConsensusAt, long TotalDocuments, long DocumentsWithConflicts);
|
||||
|
||||
private sealed record ConflictSnapshot(
|
||||
DateTimeOffset WindowStart,
|
||||
DateTimeOffset WindowEnd,
|
||||
long DocumentsWithConflicts,
|
||||
long TotalConflicts,
|
||||
IReadOnlyDictionary<string, long> ByStatus,
|
||||
IReadOnlyList<ConflictTrendPoint> Trend);
|
||||
|
||||
private sealed record SignatureSnapshot(
|
||||
DateTimeOffset WindowStart,
|
||||
DateTimeOffset WindowEnd,
|
||||
int DocumentsEvaluated,
|
||||
int WithSignatures,
|
||||
int Verified);
|
||||
}
|
||||
|
||||
internal sealed record ExcititorHealthDocument(
|
||||
DateTimeOffset GeneratedAt,
|
||||
IngestHealthSection Ingest,
|
||||
LinkHealthSection Link,
|
||||
SignatureHealthSection Signature,
|
||||
ConflictHealthSection Conflicts);
|
||||
|
||||
internal sealed record IngestHealthSection(
|
||||
string Status,
|
||||
double? MaxLagSeconds,
|
||||
IReadOnlyList<ConnectorHealth> Connectors);
|
||||
|
||||
internal sealed record ConnectorHealth(
|
||||
string ConnectorId,
|
||||
string DisplayName,
|
||||
string Status,
|
||||
DateTimeOffset? LastSuccessAt,
|
||||
DateTimeOffset? LastUpdated,
|
||||
DateTimeOffset? NextEligibleRun,
|
||||
double? LagSeconds,
|
||||
int FailureCount,
|
||||
string? LastFailureReason);
|
||||
|
||||
internal sealed record LinkHealthSection(
|
||||
string Status,
|
||||
DateTimeOffset? LastConsensusAt,
|
||||
double? LagSeconds,
|
||||
long TotalDocuments,
|
||||
long DocumentsWithConflicts);
|
||||
|
||||
internal sealed record SignatureHealthSection(
|
||||
string Status,
|
||||
DateTimeOffset WindowStart,
|
||||
DateTimeOffset WindowEnd,
|
||||
int DocumentsEvaluated,
|
||||
int WithSignatures,
|
||||
int Verified,
|
||||
int Failures,
|
||||
int Unsigned,
|
||||
double Coverage);
|
||||
|
||||
internal sealed record ConflictHealthSection(
|
||||
string Status,
|
||||
DateTimeOffset WindowStart,
|
||||
DateTimeOffset WindowEnd,
|
||||
long DocumentsWithConflicts,
|
||||
long ConflictStatements,
|
||||
IReadOnlyDictionary<string, long> ByStatus,
|
||||
IReadOnlyList<ConflictTrendPoint> Trend);
|
||||
|
||||
internal sealed record ConflictTrendPoint(DateTimeOffset BucketStart, long Conflicts);
|
||||
@@ -1,9 +1,11 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using MongoDB.Driver;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
@@ -21,43 +23,50 @@ internal interface IVexIngestOrchestrator
|
||||
Task<ReconcileSummary> ReconcileAsync(ReconcileOptions options, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
|
||||
{
|
||||
private readonly IServiceProvider _serviceProvider;
|
||||
private readonly IReadOnlyDictionary<string, IVexConnector> _connectors;
|
||||
private readonly IVexRawStore _rawStore;
|
||||
private readonly IVexClaimStore _claimStore;
|
||||
private readonly IVexProviderStore _providerStore;
|
||||
private readonly IVexConnectorStateRepository _stateRepository;
|
||||
private readonly IVexNormalizerRouter _normalizerRouter;
|
||||
private readonly IVexSignatureVerifier _signatureVerifier;
|
||||
private readonly IVexMongoSessionProvider _sessionProvider;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<VexIngestOrchestrator> _logger;
|
||||
internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
|
||||
{
|
||||
private readonly IServiceProvider _serviceProvider;
|
||||
private readonly IReadOnlyDictionary<string, IVexConnector> _connectors;
|
||||
private readonly IVexRawStore _rawStore;
|
||||
private readonly IVexClaimStore _claimStore;
|
||||
private readonly IVexProviderStore _providerStore;
|
||||
private readonly IVexConnectorStateRepository _stateRepository;
|
||||
private readonly IVexNormalizerRouter _normalizerRouter;
|
||||
private readonly IVexSignatureVerifier _signatureVerifier;
|
||||
private readonly IVexMongoSessionProvider _sessionProvider;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<VexIngestOrchestrator> _logger;
|
||||
private readonly string _defaultTenant;
|
||||
|
||||
public VexIngestOrchestrator(
|
||||
IServiceProvider serviceProvider,
|
||||
IEnumerable<IVexConnector> connectors,
|
||||
IVexRawStore rawStore,
|
||||
IVexClaimStore claimStore,
|
||||
IVexProviderStore providerStore,
|
||||
IVexConnectorStateRepository stateRepository,
|
||||
IVexNormalizerRouter normalizerRouter,
|
||||
IVexSignatureVerifier signatureVerifier,
|
||||
IVexMongoSessionProvider sessionProvider,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<VexIngestOrchestrator> logger)
|
||||
{
|
||||
_serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
|
||||
_rawStore = rawStore ?? throw new ArgumentNullException(nameof(rawStore));
|
||||
_claimStore = claimStore ?? throw new ArgumentNullException(nameof(claimStore));
|
||||
public VexIngestOrchestrator(
|
||||
IServiceProvider serviceProvider,
|
||||
IEnumerable<IVexConnector> connectors,
|
||||
IVexRawStore rawStore,
|
||||
IVexClaimStore claimStore,
|
||||
IVexProviderStore providerStore,
|
||||
IVexConnectorStateRepository stateRepository,
|
||||
IVexNormalizerRouter normalizerRouter,
|
||||
IVexSignatureVerifier signatureVerifier,
|
||||
IVexMongoSessionProvider sessionProvider,
|
||||
TimeProvider timeProvider,
|
||||
IOptions<VexMongoStorageOptions> storageOptions,
|
||||
ILogger<VexIngestOrchestrator> logger)
|
||||
{
|
||||
_serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
|
||||
_rawStore = rawStore ?? throw new ArgumentNullException(nameof(rawStore));
|
||||
_claimStore = claimStore ?? throw new ArgumentNullException(nameof(claimStore));
|
||||
_providerStore = providerStore ?? throw new ArgumentNullException(nameof(providerStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_normalizerRouter = normalizerRouter ?? throw new ArgumentNullException(nameof(normalizerRouter));
|
||||
_signatureVerifier = signatureVerifier ?? throw new ArgumentNullException(nameof(signatureVerifier));
|
||||
_sessionProvider = sessionProvider ?? throw new ArgumentNullException(nameof(sessionProvider));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_sessionProvider = sessionProvider ?? throw new ArgumentNullException(nameof(sessionProvider));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
var optionsValue = (storageOptions ?? throw new ArgumentNullException(nameof(storageOptions))).Value
|
||||
?? throw new ArgumentNullException(nameof(storageOptions));
|
||||
_defaultTenant = string.IsNullOrWhiteSpace(optionsValue.DefaultTenant)
|
||||
? "default"
|
||||
: optionsValue.DefaultTenant.Trim();
|
||||
|
||||
if (connectors is null)
|
||||
{
|
||||
@@ -149,7 +158,7 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
|
||||
|
||||
foreach (var handle in handles)
|
||||
{
|
||||
var result = await ExecuteRunAsync(handle, since, options.Force, session, cancellationToken).ConfigureAwait(false);
|
||||
var result = await ExecuteRunAsync(runId, handle, since, options.Force, session, cancellationToken).ConfigureAwait(false);
|
||||
results.Add(result);
|
||||
}
|
||||
|
||||
@@ -172,12 +181,12 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
|
||||
results.Add(ProviderRunResult.Missing(providerId, since: null));
|
||||
}
|
||||
|
||||
foreach (var handle in handles)
|
||||
{
|
||||
var since = await ResolveResumeSinceAsync(handle.Descriptor.Id, options.Checkpoint, session, cancellationToken).ConfigureAwait(false);
|
||||
var result = await ExecuteRunAsync(handle, since, force: false, session, cancellationToken).ConfigureAwait(false);
|
||||
results.Add(result);
|
||||
}
|
||||
foreach (var handle in handles)
|
||||
{
|
||||
var since = await ResolveResumeSinceAsync(handle.Descriptor.Id, options.Checkpoint, session, cancellationToken).ConfigureAwait(false);
|
||||
var result = await ExecuteRunAsync(runId, handle, since, force: false, session, cancellationToken).ConfigureAwait(false);
|
||||
results.Add(result);
|
||||
}
|
||||
|
||||
var completedAt = _timeProvider.GetUtcNow();
|
||||
return new IngestRunSummary(runId, startedAt, completedAt, results.ToImmutable());
|
||||
@@ -210,8 +219,8 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
|
||||
if (stale || state is null)
|
||||
{
|
||||
var since = stale ? threshold : lastUpdated;
|
||||
var result = await ExecuteRunAsync(handle, since, force: false, session, cancellationToken).ConfigureAwait(false);
|
||||
results.Add(new ReconcileProviderResult(
|
||||
var result = await ExecuteRunAsync(runId, handle, since, force: false, session, cancellationToken).ConfigureAwait(false);
|
||||
results.Add(new ReconcileProviderResult(
|
||||
handle.Descriptor.Id,
|
||||
result.Status,
|
||||
"reconciled",
|
||||
@@ -283,16 +292,25 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
|
||||
await _providerStore.SaveAsync(provider, cancellationToken, session).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<ProviderRunResult> ExecuteRunAsync(
|
||||
ConnectorHandle handle,
|
||||
DateTimeOffset? since,
|
||||
bool force,
|
||||
IClientSessionHandle session,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var providerId = handle.Descriptor.Id;
|
||||
var startedAt = _timeProvider.GetUtcNow();
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
private async Task<ProviderRunResult> ExecuteRunAsync(
|
||||
Guid runId,
|
||||
ConnectorHandle handle,
|
||||
DateTimeOffset? since,
|
||||
bool force,
|
||||
IClientSessionHandle session,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var providerId = handle.Descriptor.Id;
|
||||
var startedAt = _timeProvider.GetUtcNow();
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
using var scope = _logger.BeginScope(new Dictionary<string, object?>(StringComparer.Ordinal)
|
||||
{
|
||||
["tenant"] = _defaultTenant,
|
||||
["runId"] = runId,
|
||||
["providerId"] = providerId,
|
||||
["window.since"] = since?.ToString("O", CultureInfo.InvariantCulture),
|
||||
["force"] = force,
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
|
||||
@@ -7,6 +7,14 @@
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.12.0" />
|
||||
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.12.0" />
|
||||
<PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.12.0" />
|
||||
<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.12.0" />
|
||||
<PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.12.0" />
|
||||
<PackageReference Include="OpenTelemetry.Instrumentation.Runtime" Version="1.12.0" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj" />
|
||||
@@ -18,6 +26,7 @@
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Formats.CSAF/StellaOps.Excititor.Formats.CSAF.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Formats.CycloneDX/StellaOps.Excititor.Formats.CycloneDX.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Formats.OpenVEX/StellaOps.Excititor.Formats.OpenVEX.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Ingestion.Telemetry/StellaOps.Ingestion.Telemetry.csproj" />
|
||||
<ProjectReference Include="../../Aoc/__Libraries/StellaOps.Aoc/StellaOps.Aoc.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
</Project>
|
||||
|
||||
@@ -1,94 +1,95 @@
|
||||
# TASKS — Epic 1: Aggregation-Only Contract
|
||||
> **AOC Reminder:** Excititor WebService publishes raw statements/linksets only; derived precedence/severity belongs to Policy overlays.
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|---|---|---|---|---|
|
||||
| EXCITITOR-WEB-AOC-19-001 `Raw VEX ingestion APIs` | TODO | Excititor WebService Guild | EXCITITOR-CORE-AOC-19-001, EXCITITOR-STORE-AOC-19-001 | Implement `POST /ingest/vex`, `GET /vex/raw*`, and `POST /aoc/verify` endpoints. Enforce Authority scopes, tenant injection, and guard pipeline to ensure only immutable VEX facts are persisted. |
|
||||
> Docs alignment (2025-10-26): See AOC reference §4–5 and authority scopes doc for required tokens/behaviour.
|
||||
| EXCITITOR-WEB-AOC-19-002 `AOC observability + metrics` | TODO | Excititor WebService Guild, Observability Guild | EXCITITOR-WEB-AOC-19-001 | Export metrics (`ingestion_write_total`, `aoc_violation_total`, signature verification counters) and tracing spans matching Conseiller naming. Ensure structured logging includes tenant, source vendor, upstream id, and content hash. |
|
||||
> Docs alignment (2025-10-26): Metrics/traces/log schema in `docs/observability/observability.md`.
|
||||
| EXCITITOR-WEB-AOC-19-003 `Guard + schema test harness` | TODO | QA Guild | EXCITITOR-WEB-AOC-19-001 | Add unit/integration tests for schema validation, forbidden field rejection (`ERR_AOC_001/006/007`), and supersedes behavior using CycloneDX-VEX & CSAF fixtures with deterministic expectations. |
|
||||
> Docs alignment (2025-10-26): Error codes + CLI verification in `docs/modules/cli/guides/cli-reference.md`.
|
||||
| EXCITITOR-WEB-AOC-19-004 `Batch ingest validation` | TODO | Excititor WebService Guild, QA Guild | EXCITITOR-WEB-AOC-19-003, EXCITITOR-CORE-AOC-19-002 | Build large fixture ingest covering mixed VEX statuses, verifying raw storage parity, metrics, and CLI `aoc verify` compatibility. Document load test/runbook updates. |
|
||||
> Docs alignment (2025-10-26): Offline/air-gap workflows captured in `docs/deploy/containers.md` §5.
|
||||
|
||||
## Policy Engine v2
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-POLICY-20-001 `Policy selection endpoints` | TODO | Excititor WebService Guild | WEB-POLICY-20-001, EXCITITOR-CORE-AOC-19-004 | Provide VEX lookup APIs supporting PURL/advisory batching, scope filtering, and tenant enforcement with deterministic ordering + pagination. |
|
||||
|
||||
## StellaOps Console (Sprint 23)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-CONSOLE-23-001 `VEX aggregation views` | TODO | Excititor WebService Guild, BE-Base Platform Guild | EXCITITOR-LNM-21-201, EXCITITOR-LNM-21-202 | Expose `/console/vex` endpoints returning grouped VEX statements per advisory/component with status chips, justification metadata, precedence trace pointers, and tenant-scoped filters for Console explorer. |
|
||||
| EXCITITOR-CONSOLE-23-002 `Dashboard VEX deltas` | TODO | Excititor WebService Guild | EXCITITOR-CONSOLE-23-001, EXCITITOR-LNM-21-203 | Provide aggregated counts for VEX overrides (new, not_affected, revoked) powering Console dashboard + live status ticker; emit metrics for policy explain integration. |
|
||||
| EXCITITOR-CONSOLE-23-003 `VEX search helpers` | TODO | Excititor WebService Guild | EXCITITOR-CONSOLE-23-001 | Deliver rapid lookup endpoints of VEX by advisory/component for Console global search; ensure response includes provenance and precedence context; include caching and RBAC. |
|
||||
|
||||
## Graph Explorer v1
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
|
||||
## Link-Not-Merge v1
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-LNM-21-201 `Observation APIs` | TODO | Excititor WebService Guild, BE-Base Platform Guild | EXCITITOR-LNM-21-001 | Add VEX observation read endpoints with filters, pagination, RBAC, and tenant scoping. |
|
||||
| EXCITITOR-LNM-21-202 `Linkset APIs` | TODO | Excititor WebService Guild | EXCITITOR-LNM-21-002, EXCITITOR-LNM-21-003 | Implement linkset read/export/evidence endpoints returning correlation/conflict payloads and map errors to `ERR_AGG_*`. |
|
||||
| EXCITITOR-LNM-21-203 `Event publishing` | TODO | Excititor WebService Guild, Platform Events Guild | EXCITITOR-LNM-21-005 | Publish `vex.linkset.updated` events, document schema, and ensure idempotent delivery. |
|
||||
|
||||
## Graph & Vuln Explorer v1
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-GRAPH-24-101 `VEX summary API` | TODO | Excititor WebService Guild | EXCITITOR-GRAPH-24-001 | Provide endpoints delivering VEX status summaries per component/asset for Vuln Explorer integration. |
|
||||
| EXCITITOR-GRAPH-24-102 `Evidence batch API` | TODO | Excititor WebService Guild | EXCITITOR-LNM-21-201 | Add batch VEX observation retrieval optimized for Graph overlays/tooltips. |
|
||||
|
||||
## VEX Lens (Sprint 30)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-VEXLENS-30-001 `VEX evidence enrichers` | TODO | Excititor WebService Guild, VEX Lens Guild | EXCITITOR-VULN-29-001, VEXLENS-30-005 | Include issuer hints, signatures, and product trees in evidence payloads for VEX Lens; Label: VEX-Lens. |
|
||||
|
||||
## Vulnerability Explorer (Sprint 29)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-VULN-29-001 `VEX key canonicalization` | TODO | Excititor WebService Guild | EXCITITOR-LNM-21-001 | Canonicalize (lossless) VEX advisory/product keys (map to `advisory_key`, capture product scopes); expose original sources in `links[]`; AOC-compliant: no merge, no derived fields, no suppression; backfill existing records. |
|
||||
| EXCITITOR-VULN-29-002 `Evidence retrieval` | TODO | Excititor WebService Guild | EXCITITOR-VULN-29-001, VULN-API-29-003 | Provide `/vuln/evidence/vex/{advisory_key}` returning raw VEX statements filtered by tenant/product scope for Explorer evidence tabs. |
|
||||
| EXCITITOR-VULN-29-004 `Observability` | TODO | Excititor WebService Guild, Observability Guild | EXCITITOR-VULN-29-001 | Add metrics/logs for VEX normalization, suppression scopes, withdrawn statements; emit events consumed by Vuln Explorer resolver. |
|
||||
|
||||
## Advisory AI (Sprint 31)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-AIAI-31-001 `Justification enrichment` | TODO | Excititor WebService Guild | EXCITITOR-VULN-29-001 | Expose normalized VEX justifications, product trees, and paragraph anchors for Advisory AI conflict explanations. |
|
||||
| EXCITITOR-AIAI-31-002 `VEX chunk API` | TODO | Excititor WebService Guild | EXCITITOR-AIAI-31-001, VEXLENS-30-006 | Provide `/vex/evidence/chunks` endpoint returning tenant-scoped VEX statements with signature metadata and scope scores for RAG. |
|
||||
| EXCITITOR-AIAI-31-003 `Telemetry` | TODO | Excititor WebService Guild, Observability Guild | EXCITITOR-AIAI-31-001 | Emit metrics/logs for VEX chunk usage, signature verification failures, and guardrail triggers. |
|
||||
|
||||
## Observability & Forensics (Epic 15)
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-WEB-OBS-50-001 `Telemetry adoption` | TODO | Excititor WebService Guild | TELEMETRY-OBS-50-001, EXCITITOR-OBS-50-001 | Adopt telemetry core for VEX APIs, ensure responses include trace IDs & correlation headers, and update structured logging for read endpoints. |
|
||||
| EXCITITOR-WEB-OBS-51-001 `Observability health endpoints` | TODO | Excititor WebService Guild | EXCITITOR-WEB-OBS-50-001, WEB-OBS-51-001 | Implement `/obs/excititor/health` summarizing ingest/link SLOs, signature failure counts, and conflict trends for Console dashboards. |
|
||||
| EXCITITOR-WEB-OBS-52-001 `Timeline streaming` | TODO | Excititor WebService Guild | EXCITITOR-WEB-OBS-50-001, TIMELINE-OBS-52-003 | Provide SSE bridge for VEX timeline events with tenant filters, pagination, and guardrails. |
|
||||
| EXCITITOR-WEB-OBS-53-001 `Evidence APIs` | TODO | Excititor WebService Guild, Evidence Locker Guild | EXCITITOR-OBS-53-001, EVID-OBS-53-003 | Expose `/evidence/vex/*` endpoints that fetch locker bundles, enforce scopes, and surface verification metadata. |
|
||||
| EXCITITOR-WEB-OBS-54-001 `Attestation APIs` | TODO | Excititor WebService Guild | EXCITITOR-OBS-54-001, PROV-OBS-54-001 | Add `/attestations/vex/*` endpoints returning DSSE verification state, builder identity, and chain-of-custody links. |
|
||||
| EXCITITOR-WEB-OBS-55-001 `Incident mode toggles` | TODO | Excititor WebService Guild, DevOps Guild | EXCITITOR-OBS-55-001, WEB-OBS-55-001 | Provide incident mode API for VEX pipelines with activation audit logs and retention override previews. |
|
||||
|
||||
## Air-Gapped Mode (Epic 16)
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-WEB-AIRGAP-56-001 | TODO | Excititor WebService Guild | AIRGAP-IMP-58-001, EXCITITOR-AIRGAP-56-001 | Support mirror bundle registration via APIs, expose bundle provenance in VEX responses, and block external connectors in sealed mode. |
|
||||
| EXCITITOR-WEB-AIRGAP-56-002 | TODO | Excititor WebService Guild, AirGap Time Guild | EXCITITOR-WEB-AIRGAP-56-001, AIRGAP-TIME-58-001 | Return VEX staleness metrics and time anchor info in API responses for Console/CLI use. |
|
||||
| EXCITITOR-WEB-AIRGAP-57-001 | TODO | Excititor WebService Guild, AirGap Policy Guild | AIRGAP-POL-56-001 | Map sealed-mode violations to standardized error payload with remediation guidance. |
|
||||
| EXCITITOR-WEB-AIRGAP-58-001 | TODO | Excititor WebService Guild, AirGap Importer Guild | EXCITITOR-WEB-AIRGAP-56-001, TIMELINE-OBS-53-001 | Emit timeline events for VEX bundle imports with bundle ID, scope, and actor metadata. |
|
||||
|
||||
## SDKs & OpenAPI (Epic 17)
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-WEB-OAS-61-001 | TODO | Excititor WebService Guild | OAS-61-001 | Implement `/.well-known/openapi` discovery endpoint with spec version metadata. |
|
||||
| EXCITITOR-WEB-OAS-61-002 | TODO | Excititor WebService Guild | APIGOV-61-001 | Standardize error envelope responses and update controller/unit tests. |
|
||||
| EXCITITOR-WEB-OAS-62-001 | TODO | Excititor WebService Guild | EXCITITOR-OAS-61-002 | Add curated examples for VEX observation/linkset endpoints and ensure portal displays them. |
|
||||
| EXCITITOR-WEB-OAS-63-001 | TODO | Excititor WebService Guild, API Governance Guild | APIGOV-63-001 | Emit deprecation headers and update docs for retiring VEX APIs. |
|
||||
# TASKS — Epic 1: Aggregation-Only Contract
|
||||
> **AOC Reminder:** Excititor WebService publishes raw statements/linksets only; derived precedence/severity belongs to Policy overlays.
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|---|---|---|---|---|
|
||||
| EXCITITOR-WEB-AOC-19-001 `Raw VEX ingestion APIs` | DONE (2025-11-08) | Excititor WebService Guild | EXCITITOR-CORE-AOC-19-001, EXCITITOR-STORE-AOC-19-001 | Implement `POST /ingest/vex`, `GET /vex/raw*`, and `POST /aoc/verify` endpoints. Enforce Authority scopes, tenant injection, and guard pipeline to ensure only immutable VEX facts are persisted. |
|
||||
> Docs alignment (2025-10-26): See AOC reference §4–5 and authority scopes doc for required tokens/behaviour.
|
||||
| EXCITITOR-WEB-AOC-19-002 `AOC observability + metrics` | DONE (2025-11-08) | Excititor WebService Guild, Observability Guild | EXCITITOR-WEB-AOC-19-001 | Export metrics (`ingestion_write_total`, `aoc_violation_total`, signature verification counters) and tracing spans matching Conseiller naming. Ensure structured logging includes tenant, source vendor, upstream id, and content hash. |
|
||||
> Docs alignment (2025-10-26): Metrics/traces/log schema in `docs/observability/observability.md`.
|
||||
| EXCITITOR-WEB-AOC-19-003 `Guard + schema test harness` | DONE (2025-11-08) | QA Guild | EXCITITOR-WEB-AOC-19-001 | Add unit/integration tests for schema validation, forbidden field rejection (`ERR_AOC_001/006/007`), and supersedes behavior using CycloneDX-VEX & CSAF fixtures with deterministic expectations. |
|
||||
> Docs alignment (2025-10-26): Error codes + CLI verification in `docs/modules/cli/guides/cli-reference.md`.
|
||||
| EXCITITOR-WEB-AOC-19-004 `Batch ingest validation` | DONE (2025-11-08) | Excititor WebService Guild, QA Guild | EXCITITOR-WEB-AOC-19-003, EXCITITOR-CORE-AOC-19-002 | Build large fixture ingest covering mixed VEX statuses, verifying raw storage parity, metrics, and CLI `aoc verify` compatibility. Document load test/runbook updates. |
|
||||
> Docs alignment (2025-10-26): Offline/air-gap workflows captured in `docs/deploy/containers.md` §5.
|
||||
| EXCITITOR-CRYPTO-90-001 `Crypto provider adoption` | TODO | Excititor WebService Guild, Security Guild | SEC-CRYPTO-90-003, SEC-CRYPTO-90-004 | Replace direct `System.Security.Cryptography` hashing/signing inside connector loaders, VEX exporters, and OpenAPI discovery with `ICryptoProviderRegistry` + `ICryptoHash`. Reference `docs/security/crypto-routing-audit-2025-11-07.md`. | Registry-backed providers configurable per deployment; integration tests cover default + `ru-offline` profiles; connectors honor sovereign provider ordering. |
|
||||
|
||||
## Policy Engine v2
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-POLICY-20-001 `Policy selection endpoints` | TODO | Excititor WebService Guild | WEB-POLICY-20-001, EXCITITOR-CORE-AOC-19-004 | Provide VEX lookup APIs supporting PURL/advisory batching, scope filtering, and tenant enforcement with deterministic ordering + pagination. |
|
||||
|
||||
## StellaOps Console (Sprint 23)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-CONSOLE-23-001 `VEX aggregation views` | TODO | Excititor WebService Guild, BE-Base Platform Guild | EXCITITOR-LNM-21-201, EXCITITOR-LNM-21-202 | Expose `/console/vex` endpoints returning grouped VEX statements per advisory/component with status chips, justification metadata, precedence trace pointers, and tenant-scoped filters for Console explorer. |
|
||||
| EXCITITOR-CONSOLE-23-002 `Dashboard VEX deltas` | TODO | Excititor WebService Guild | EXCITITOR-CONSOLE-23-001, EXCITITOR-LNM-21-203 | Provide aggregated counts for VEX overrides (new, not_affected, revoked) powering Console dashboard + live status ticker; emit metrics for policy explain integration. |
|
||||
| EXCITITOR-CONSOLE-23-003 `VEX search helpers` | TODO | Excititor WebService Guild | EXCITITOR-CONSOLE-23-001 | Deliver rapid lookup endpoints of VEX by advisory/component for Console global search; ensure response includes provenance and precedence context; include caching and RBAC. |
|
||||
|
||||
## Graph Explorer v1
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
|
||||
## Link-Not-Merge v1
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-LNM-21-201 `Observation APIs` | TODO | Excititor WebService Guild, BE-Base Platform Guild | EXCITITOR-LNM-21-001 | Add VEX observation read endpoints with filters, pagination, RBAC, and tenant scoping. |
|
||||
| EXCITITOR-LNM-21-202 `Linkset APIs` | TODO | Excititor WebService Guild | EXCITITOR-LNM-21-002, EXCITITOR-LNM-21-003 | Implement linkset read/export/evidence endpoints returning correlation/conflict payloads and map errors to `ERR_AGG_*`. |
|
||||
| EXCITITOR-LNM-21-203 `Event publishing` | TODO | Excititor WebService Guild, Platform Events Guild | EXCITITOR-LNM-21-005 | Publish `vex.linkset.updated` events, document schema, and ensure idempotent delivery. |
|
||||
|
||||
## Graph & Vuln Explorer v1
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-GRAPH-24-101 `VEX summary API` | TODO | Excititor WebService Guild | EXCITITOR-GRAPH-24-001 | Provide endpoints delivering VEX status summaries per component/asset for Vuln Explorer integration. |
|
||||
| EXCITITOR-GRAPH-24-102 `Evidence batch API` | TODO | Excititor WebService Guild | EXCITITOR-LNM-21-201 | Add batch VEX observation retrieval optimized for Graph overlays/tooltips. |
|
||||
|
||||
## VEX Lens (Sprint 30)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-VEXLENS-30-001 `VEX evidence enrichers` | TODO | Excititor WebService Guild, VEX Lens Guild | EXCITITOR-VULN-29-001, VEXLENS-30-005 | Include issuer hints, signatures, and product trees in evidence payloads for VEX Lens; Label: VEX-Lens. |
|
||||
|
||||
## Vulnerability Explorer (Sprint 29)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-VULN-29-001 `VEX key canonicalization` | TODO | Excititor WebService Guild | EXCITITOR-LNM-21-001 | Canonicalize (lossless) VEX advisory/product keys (map to `advisory_key`, capture product scopes); expose original sources in `links[]`; AOC-compliant: no merge, no derived fields, no suppression; backfill existing records. |
|
||||
| EXCITITOR-VULN-29-002 `Evidence retrieval` | TODO | Excititor WebService Guild | EXCITITOR-VULN-29-001, VULN-API-29-003 | Provide `/vuln/evidence/vex/{advisory_key}` returning raw VEX statements filtered by tenant/product scope for Explorer evidence tabs. |
|
||||
| EXCITITOR-VULN-29-004 `Observability` | TODO | Excititor WebService Guild, Observability Guild | EXCITITOR-VULN-29-001 | Add metrics/logs for VEX normalization, suppression scopes, withdrawn statements; emit events consumed by Vuln Explorer resolver. |
|
||||
|
||||
## Advisory AI (Sprint 31)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-AIAI-31-001 `Justification enrichment` | TODO | Excititor WebService Guild | EXCITITOR-VULN-29-001 | Expose normalized VEX justifications, product trees, and paragraph anchors for Advisory AI conflict explanations. |
|
||||
| EXCITITOR-AIAI-31-002 `VEX chunk API` | TODO | Excititor WebService Guild | EXCITITOR-AIAI-31-001, VEXLENS-30-006 | Provide `/vex/evidence/chunks` endpoint returning tenant-scoped VEX statements with signature metadata and scope scores for RAG. |
|
||||
| EXCITITOR-AIAI-31-003 `Telemetry` | TODO | Excititor WebService Guild, Observability Guild | EXCITITOR-AIAI-31-001 | Emit metrics/logs for VEX chunk usage, signature verification failures, and guardrail triggers. |
|
||||
|
||||
## Observability & Forensics (Epic 15)
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-WEB-OBS-50-001 `Telemetry adoption` | DONE (2025-11-07) | Excititor WebService Guild | TELEMETRY-OBS-50-001, EXCITITOR-OBS-50-001 | Adopt telemetry core for VEX APIs, ensure responses include trace IDs & correlation headers, and update structured logging for read endpoints. |
|
||||
| EXCITITOR-WEB-OBS-51-001 `Observability health endpoints` | DONE (2025-11-08) | Excititor WebService Guild | EXCITITOR-WEB-OBS-50-001, WEB-OBS-51-001 | Implement `/obs/excititor/health` summarizing ingest/link SLOs, signature failure counts, and conflict trends for Console dashboards. |
|
||||
| EXCITITOR-WEB-OBS-52-001 `Timeline streaming` | TODO | Excititor WebService Guild | EXCITITOR-WEB-OBS-50-001, TIMELINE-OBS-52-003 | Provide SSE bridge for VEX timeline events with tenant filters, pagination, and guardrails. |
|
||||
| EXCITITOR-WEB-OBS-53-001 `Evidence APIs` | TODO | Excititor WebService Guild, Evidence Locker Guild | EXCITITOR-OBS-53-001, EVID-OBS-53-003 | Expose `/evidence/vex/*` endpoints that fetch locker bundles, enforce scopes, and surface verification metadata. |
|
||||
| EXCITITOR-WEB-OBS-54-001 `Attestation APIs` | TODO | Excititor WebService Guild | EXCITITOR-OBS-54-001, PROV-OBS-54-001 | Add `/attestations/vex/*` endpoints returning DSSE verification state, builder identity, and chain-of-custody links. |
|
||||
| EXCITITOR-WEB-OBS-55-001 `Incident mode toggles` | TODO | Excititor WebService Guild, DevOps Guild | EXCITITOR-OBS-55-001, WEB-OBS-55-001 | Provide incident mode API for VEX pipelines with activation audit logs and retention override previews. |
|
||||
|
||||
## Air-Gapped Mode (Epic 16)
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-WEB-AIRGAP-56-001 | TODO | Excititor WebService Guild | AIRGAP-IMP-58-001, EXCITITOR-AIRGAP-56-001 | Support mirror bundle registration via APIs, expose bundle provenance in VEX responses, and block external connectors in sealed mode. |
|
||||
| EXCITITOR-WEB-AIRGAP-56-002 | TODO | Excititor WebService Guild, AirGap Time Guild | EXCITITOR-WEB-AIRGAP-56-001, AIRGAP-TIME-58-001 | Return VEX staleness metrics and time anchor info in API responses for Console/CLI use. |
|
||||
| EXCITITOR-WEB-AIRGAP-57-001 | TODO | Excititor WebService Guild, AirGap Policy Guild | AIRGAP-POL-56-001 | Map sealed-mode violations to standardized error payload with remediation guidance. |
|
||||
| EXCITITOR-WEB-AIRGAP-58-001 | TODO | Excititor WebService Guild, AirGap Importer Guild | EXCITITOR-WEB-AIRGAP-56-001, TIMELINE-OBS-53-001 | Emit timeline events for VEX bundle imports with bundle ID, scope, and actor metadata. |
|
||||
|
||||
## SDKs & OpenAPI (Epic 17)
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-WEB-OAS-61-001 | TODO | Excititor WebService Guild | OAS-61-001 | Implement `/.well-known/openapi` discovery endpoint with spec version metadata. |
|
||||
| EXCITITOR-WEB-OAS-61-002 | TODO | Excititor WebService Guild | APIGOV-61-001 | Standardize error envelope responses and update controller/unit tests. |
|
||||
| EXCITITOR-WEB-OAS-62-001 | TODO | Excititor WebService Guild | EXCITITOR-OAS-61-002 | Add curated examples for VEX observation/linkset endpoints and ensure portal displays them. |
|
||||
| EXCITITOR-WEB-OAS-63-001 | TODO | Excititor WebService Guild, API Governance Guild | APIGOV-63-001 | Emit deprecation headers and update docs for retiring VEX APIs. |
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Aoc;
|
||||
using RawVexDocument = StellaOps.Concelier.RawModels.VexRawDocument;
|
||||
using System.Diagnostics;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Aoc;
|
||||
using RawVexDocument = StellaOps.Concelier.RawModels.VexRawDocument;
|
||||
using StellaOps.Ingestion.Telemetry;
|
||||
|
||||
namespace StellaOps.Excititor.Core.Aoc;
|
||||
|
||||
@@ -21,15 +23,35 @@ public sealed class VexRawWriteGuard : IVexRawWriteGuard
|
||||
_options = options?.Value ?? AocGuardOptions.Default;
|
||||
}
|
||||
|
||||
public void EnsureValid(RawVexDocument document)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
using var payload = JsonDocument.Parse(JsonSerializer.Serialize(document, SerializerOptions));
|
||||
var result = _guard.Validate(payload.RootElement, _options);
|
||||
if (!result.IsValid)
|
||||
{
|
||||
throw new ExcititorAocGuardException(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
public void EnsureValid(RawVexDocument document)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
using var guardActivity = IngestionTelemetry.StartGuardActivity(
|
||||
document.Tenant,
|
||||
document.Source.Vendor,
|
||||
document.Upstream.UpstreamId,
|
||||
document.Upstream.ContentHash,
|
||||
document.Supersedes);
|
||||
|
||||
using var payload = JsonDocument.Parse(JsonSerializer.Serialize(document, SerializerOptions));
|
||||
var result = _guard.Validate(payload.RootElement, _options);
|
||||
if (!result.IsValid)
|
||||
{
|
||||
var violationCount = result.Violations.IsDefaultOrEmpty ? 0 : result.Violations.Length;
|
||||
var primaryCode = violationCount > 0 ? result.Violations[0].ErrorCode : string.Empty;
|
||||
|
||||
guardActivity?.SetTag("violationCount", violationCount);
|
||||
if (!string.IsNullOrWhiteSpace(primaryCode))
|
||||
{
|
||||
guardActivity?.SetTag("code", primaryCode);
|
||||
}
|
||||
|
||||
guardActivity?.SetStatus(ActivityStatusCode.Error, primaryCode);
|
||||
throw new ExcititorAocGuardException(result);
|
||||
}
|
||||
|
||||
guardActivity?.SetTag("violationCount", 0);
|
||||
guardActivity?.SetStatus(ActivityStatusCode.Ok);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,5 +13,6 @@
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../../Aoc/__Libraries/StellaOps.Aoc/StellaOps.Aoc.csproj" />
|
||||
<ProjectReference Include="../../../Concelier/__Libraries/StellaOps.Concelier.RawModels/StellaOps.Concelier.RawModels.csproj" />
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.Ingestion.Telemetry/StellaOps.Ingestion.Telemetry.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -62,7 +62,7 @@
|
||||
## Observability & Forensics (Epic 15)
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-OBS-50-001 `Telemetry adoption` | TODO | Excititor Core Guild, Observability Guild | TELEMETRY-OBS-50-001, TELEMETRY-OBS-50-002 | Integrate telemetry core across VEX ingestion/linking, ensuring spans/logs capture tenant, product scope, upstream id, justification hash, and trace IDs. |
|
||||
| EXCITITOR-OBS-50-001 `Telemetry adoption` | DONE (2025-11-07) | Excititor Core Guild, Observability Guild | TELEMETRY-OBS-50-001, TELEMETRY-OBS-50-002 | Integrate telemetry core across VEX ingestion/linking, ensuring spans/logs capture tenant, product scope, upstream id, justification hash, and trace IDs. |
|
||||
| EXCITITOR-OBS-51-001 `Metrics & SLOs` | TODO | Excititor Core Guild, DevOps Guild | EXCITITOR-OBS-50-001, TELEMETRY-OBS-51-001 | Publish metrics for VEX ingest latency, scope resolution success, conflict rate, signature verification failures. Define SLOs (link latency P95 <30s) and configure burn-rate alerts. |
|
||||
| EXCITITOR-OBS-52-001 `Timeline events` | TODO | Excititor Core Guild | EXCITITOR-OBS-50-001, TIMELINE-OBS-52-002 | Emit `timeline_event` entries for VEX ingest/linking/outcome changes with trace IDs, justification summaries, and evidence placeholders. |
|
||||
| EXCITITOR-OBS-53-001 `Evidence snapshots` | TODO | Excititor Core Guild, Evidence Locker Guild | EXCITITOR-OBS-52-001, EVID-OBS-53-002 | Build evidence payloads for VEX statements (raw doc, normalization diff, precedence notes) and push to evidence locker with Merkle manifests. |
|
||||
|
||||
@@ -68,6 +68,8 @@ public interface IVexConnectorStateRepository
|
||||
ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
}
|
||||
|
||||
public interface IVexConsensusHoldStore
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
using System;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
@@ -29,11 +30,11 @@ public sealed class MongoVexConnectorStateRepository : IVexConnectorStateReposit
|
||||
return document?.ToRecord();
|
||||
}
|
||||
|
||||
public async ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(state);
|
||||
|
||||
var document = VexConnectorStateDocument.FromRecord(state.WithNormalizedDigests());
|
||||
public async ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(state);
|
||||
|
||||
var document = VexConnectorStateDocument.FromRecord(state.WithNormalizedDigests());
|
||||
var filter = Builders<VexConnectorStateDocument>.Filter.Eq(x => x.ConnectorId, document.ConnectorId);
|
||||
if (session is null)
|
||||
{
|
||||
@@ -41,10 +42,24 @@ public sealed class MongoVexConnectorStateRepository : IVexConnectorStateReposit
|
||||
}
|
||||
else
|
||||
{
|
||||
await _collection.ReplaceOneAsync(session, filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
await _collection.ReplaceOneAsync(session, filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
var find = session is null
|
||||
? _collection.Find(FilterDefinition<VexConnectorStateDocument>.Empty)
|
||||
: _collection.Find(session, FilterDefinition<VexConnectorStateDocument>.Empty);
|
||||
|
||||
var documents = await find
|
||||
.SortBy(x => x.ConnectorId)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return documents.ConvertAll(static document => document.ToRecord());
|
||||
}
|
||||
}
|
||||
|
||||
internal static class VexConnectorStateExtensions
|
||||
{
|
||||
|
||||
@@ -1,36 +1,34 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using MongoDB.Driver.Core.Clusters;
|
||||
using MongoDB.Driver.Core.Clusters;
|
||||
using MongoDB.Driver.GridFS;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Aoc;
|
||||
using RawContentMetadata = StellaOps.Concelier.RawModels.RawContent;
|
||||
using RawLinkset = StellaOps.Concelier.RawModels.RawLinkset;
|
||||
using RawReference = StellaOps.Concelier.RawModels.RawReference;
|
||||
using RawSignatureMetadata = StellaOps.Concelier.RawModels.RawSignatureMetadata;
|
||||
using RawSourceMetadata = StellaOps.Concelier.RawModels.RawSourceMetadata;
|
||||
using RawUpstreamMetadata = StellaOps.Concelier.RawModels.RawUpstreamMetadata;
|
||||
using RawVexDocument = StellaOps.Concelier.RawModels.VexRawDocument;
|
||||
using VexStatementSummaryModel = StellaOps.Concelier.RawModels.VexStatementSummary;
|
||||
using StellaOps.Ingestion.Telemetry;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
public sealed class MongoVexRawStore : IVexRawStore
|
||||
{
|
||||
private readonly IMongoClient _client;
|
||||
private readonly IMongoClient _client;
|
||||
private readonly IMongoCollection<VexRawDocumentRecord> _collection;
|
||||
private readonly GridFSBucket _bucket;
|
||||
private readonly VexMongoStorageOptions _options;
|
||||
private readonly IVexMongoSessionProvider _sessionProvider;
|
||||
private readonly IVexRawWriteGuard _guard;
|
||||
private readonly ILogger<MongoVexRawStore> _logger;
|
||||
private readonly string _connectorVersion;
|
||||
|
||||
public MongoVexRawStore(
|
||||
@@ -38,13 +36,15 @@ public sealed class MongoVexRawStore : IVexRawStore
|
||||
IMongoDatabase database,
|
||||
IOptions<VexMongoStorageOptions> options,
|
||||
IVexMongoSessionProvider sessionProvider,
|
||||
IVexRawWriteGuard guard)
|
||||
IVexRawWriteGuard guard,
|
||||
ILogger<MongoVexRawStore>? logger = null)
|
||||
{
|
||||
_client = client ?? throw new ArgumentNullException(nameof(client));
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
_sessionProvider = sessionProvider ?? throw new ArgumentNullException(nameof(sessionProvider));
|
||||
_guard = guard ?? throw new ArgumentNullException(nameof(guard));
|
||||
_logger = logger ?? NullLogger<MongoVexRawStore>.Instance;
|
||||
|
||||
_options = options.Value;
|
||||
Validator.ValidateObject(_options, new ValidationContext(_options), validateAllProperties: true);
|
||||
@@ -54,350 +54,285 @@ public sealed class MongoVexRawStore : IVexRawStore
|
||||
_collection = database.GetCollection<VexRawDocumentRecord>(VexMongoCollectionNames.Raw);
|
||||
_bucket = new GridFSBucket(database, new GridFSBucketOptions
|
||||
{
|
||||
BucketName = _options.RawBucketName,
|
||||
ReadConcern = database.Settings.ReadConcern,
|
||||
ReadPreference = database.Settings.ReadPreference,
|
||||
WriteConcern = database.Settings.WriteConcern,
|
||||
});
|
||||
}
|
||||
|
||||
BucketName = _options.RawBucketName,
|
||||
ReadConcern = database.Settings.ReadConcern,
|
||||
ReadPreference = database.Settings.ReadPreference,
|
||||
WriteConcern = database.Settings.WriteConcern,
|
||||
});
|
||||
}
|
||||
|
||||
public async ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var guardPayload = CreateRawModel(document);
|
||||
_guard.EnsureValid(guardPayload);
|
||||
var guardPayload = VexRawDocumentMapper.ToRawModel(document, _options.DefaultTenant);
|
||||
var tenant = guardPayload.Tenant;
|
||||
var sourceVendor = guardPayload.Source.Vendor;
|
||||
var upstreamId = guardPayload.Upstream.UpstreamId;
|
||||
var contentHash = guardPayload.Upstream.ContentHash;
|
||||
|
||||
using var logScope = _logger.BeginScope(new Dictionary<string, object?>(StringComparer.Ordinal)
|
||||
{
|
||||
["tenant"] = tenant,
|
||||
["source.vendor"] = sourceVendor,
|
||||
["upstream.upstreamId"] = upstreamId,
|
||||
["contentHash"] = contentHash,
|
||||
["providerId"] = document.ProviderId,
|
||||
["digest"] = document.Digest,
|
||||
});
|
||||
|
||||
var transformWatch = Stopwatch.StartNew();
|
||||
using var transformActivity = IngestionTelemetry.StartTransformActivity(
|
||||
tenant,
|
||||
sourceVendor,
|
||||
upstreamId,
|
||||
contentHash,
|
||||
document.Format.ToString(),
|
||||
document.Content.Length);
|
||||
|
||||
try
|
||||
{
|
||||
_guard.EnsureValid(guardPayload);
|
||||
transformActivity?.SetStatus(ActivityStatusCode.Ok);
|
||||
}
|
||||
catch (ExcititorAocGuardException ex)
|
||||
{
|
||||
transformActivity?.SetTag("violationCount", ex.Violations.IsDefaultOrEmpty ? 0 : ex.Violations.Length);
|
||||
transformActivity?.SetTag("code", ex.PrimaryErrorCode);
|
||||
transformActivity?.SetStatus(ActivityStatusCode.Error, ex.PrimaryErrorCode);
|
||||
|
||||
IngestionTelemetry.RecordViolation(tenant, sourceVendor, ex.PrimaryErrorCode);
|
||||
IngestionTelemetry.RecordWriteAttempt(tenant, sourceVendor, IngestionTelemetry.ResultReject);
|
||||
|
||||
_logger.LogWarning(ex, "AOC guard rejected VEX document digest={Digest} provider={ProviderId}", document.Digest, document.ProviderId);
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (transformWatch.IsRunning)
|
||||
{
|
||||
transformWatch.Stop();
|
||||
}
|
||||
|
||||
IngestionTelemetry.RecordLatency(tenant, sourceVendor, IngestionTelemetry.PhaseTransform, transformWatch.Elapsed);
|
||||
}
|
||||
|
||||
var threshold = _options.GridFsInlineThresholdBytes;
|
||||
var useInline = threshold == 0 || document.Content.Length <= threshold;
|
||||
string? newGridId = null;
|
||||
string? oldGridIdToDelete = null;
|
||||
|
||||
var sessionHandle = session ?? await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!useInline)
|
||||
{
|
||||
newGridId = await UploadToGridFsAsync(document, sessionHandle, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var supportsTransactions = sessionHandle.Client.Cluster.Description.Type != ClusterType.Standalone
|
||||
&& !sessionHandle.IsInTransaction;
|
||||
|
||||
var startedTransaction = false;
|
||||
if (supportsTransactions)
|
||||
{
|
||||
try
|
||||
{
|
||||
sessionHandle.StartTransaction();
|
||||
startedTransaction = true;
|
||||
}
|
||||
catch (NotSupportedException)
|
||||
{
|
||||
supportsTransactions = false;
|
||||
}
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var filter = Builders<VexRawDocumentRecord>.Filter.Eq(x => x.Id, document.Digest);
|
||||
var existing = await _collection
|
||||
.Find(sessionHandle, filter)
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var record = VexRawDocumentRecord.FromDomain(document, includeContent: useInline);
|
||||
record.GridFsObjectId = useInline ? null : newGridId;
|
||||
|
||||
await _collection
|
||||
.ReplaceOneAsync(
|
||||
sessionHandle,
|
||||
filter,
|
||||
record,
|
||||
new ReplaceOptions { IsUpsert = true },
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (existing?.GridFsObjectId is string oldGridId && !string.IsNullOrWhiteSpace(oldGridId))
|
||||
{
|
||||
if (useInline || !string.Equals(newGridId, oldGridId, StringComparison.Ordinal))
|
||||
{
|
||||
oldGridIdToDelete = oldGridId;
|
||||
}
|
||||
}
|
||||
|
||||
if (startedTransaction)
|
||||
{
|
||||
await sessionHandle.CommitTransactionAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
if (startedTransaction && sessionHandle.IsInTransaction)
|
||||
{
|
||||
await sessionHandle.AbortTransactionAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (!useInline && !string.IsNullOrWhiteSpace(newGridId))
|
||||
{
|
||||
await DeleteFromGridFsAsync(newGridId, sessionHandle, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
throw;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(oldGridIdToDelete))
|
||||
{
|
||||
await DeleteFromGridFsAsync(oldGridIdToDelete!, sessionHandle, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<VexRawDocument?> FindByDigestAsync(string digest, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
throw new ArgumentException("Digest must be provided.", nameof(digest));
|
||||
}
|
||||
|
||||
var trimmed = digest.Trim();
|
||||
var filter = Builders<VexRawDocumentRecord>.Filter.Eq(x => x.Id, trimmed);
|
||||
var record = session is null
|
||||
? await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false)
|
||||
: await _collection.Find(session, filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (record is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(record.GridFsObjectId))
|
||||
{
|
||||
var handle = session ?? await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
|
||||
var bytes = await DownloadFromGridFsAsync(record.GridFsObjectId, handle, cancellationToken).ConfigureAwait(false);
|
||||
return record.ToDomain(new ReadOnlyMemory<byte>(bytes));
|
||||
}
|
||||
|
||||
return record.ToDomain();
|
||||
}
|
||||
|
||||
private async Task<string?> UploadToGridFsAsync(VexRawDocument document, IClientSessionHandle? session, CancellationToken cancellationToken)
|
||||
{
|
||||
using var stream = new MemoryStream(document.Content.ToArray(), writable: false);
|
||||
var metadata = new BsonDocument
|
||||
{
|
||||
{ "providerId", document.ProviderId },
|
||||
{ "format", document.Format.ToString().ToLowerInvariant() },
|
||||
{ "sourceUri", document.SourceUri.ToString() },
|
||||
{ "retrievedAt", document.RetrievedAt.UtcDateTime },
|
||||
};
|
||||
|
||||
var options = new GridFSUploadOptions { Metadata = metadata };
|
||||
var objectId = await _bucket
|
||||
.UploadFromStreamAsync(document.Digest, stream, options, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return objectId.ToString();
|
||||
}
|
||||
|
||||
private async Task DeleteFromGridFsAsync(string gridFsObjectId, IClientSessionHandle? session, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!ObjectId.TryParse(gridFsObjectId, out var objectId))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await _bucket.DeleteAsync(objectId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (GridFSFileNotFoundException)
|
||||
{
|
||||
// file already removed by TTL or manual cleanup
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<byte[]> DownloadFromGridFsAsync(string gridFsObjectId, IClientSessionHandle? session, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!ObjectId.TryParse(gridFsObjectId, out var objectId))
|
||||
{
|
||||
return Array.Empty<byte>();
|
||||
}
|
||||
|
||||
return await _bucket.DownloadAsBytesAsync(objectId, null, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
async ValueTask IVexRawDocumentSink.StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> await StoreAsync(document, cancellationToken, session: null).ConfigureAwait(false);
|
||||
var sessionHandle = session ?? await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
private RawVexDocument CreateRawModel(VexRawDocument document)
|
||||
{
|
||||
var metadata = document.Metadata ?? ImmutableDictionary<string, string>.Empty;
|
||||
var tenant = _options.DefaultTenant;
|
||||
if (!useInline)
|
||||
{
|
||||
newGridId = await UploadToGridFsAsync(document, sessionHandle, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var source = CreateSourceMetadata(document, metadata);
|
||||
var content = CreateContent(document, metadata);
|
||||
var upstream = CreateUpstreamMetadata(document, metadata);
|
||||
var linkset = CreateLinkset();
|
||||
var statements = ImmutableArray<VexStatementSummaryModel>.Empty;
|
||||
var supportsTransactions = sessionHandle.Client.Cluster.Description.Type != ClusterType.Standalone
|
||||
&& !sessionHandle.IsInTransaction;
|
||||
|
||||
return new RawVexDocument(
|
||||
var startedTransaction = false;
|
||||
if (supportsTransactions)
|
||||
{
|
||||
try
|
||||
{
|
||||
sessionHandle.StartTransaction();
|
||||
startedTransaction = true;
|
||||
}
|
||||
catch (NotSupportedException)
|
||||
{
|
||||
supportsTransactions = false;
|
||||
}
|
||||
}
|
||||
|
||||
var fetchWatch = Stopwatch.StartNew();
|
||||
using var fetchActivity = IngestionTelemetry.StartFetchActivity(
|
||||
tenant,
|
||||
source,
|
||||
upstream,
|
||||
content,
|
||||
linkset,
|
||||
statements);
|
||||
}
|
||||
|
||||
private RawSourceMetadata CreateSourceMetadata(VexRawDocument document, ImmutableDictionary<string, string> metadata)
|
||||
{
|
||||
var vendor = TryMetadata(metadata, "source.vendor", "connector.vendor") ?? ExtractVendor(document.ProviderId);
|
||||
var connector = TryMetadata(metadata, "source.connector") ?? document.ProviderId;
|
||||
var version = TryMetadata(metadata, "source.connector_version", "connector.version") ?? _connectorVersion;
|
||||
var stream = TryMetadata(metadata, "source.stream", "connector.stream") ?? document.Format.ToString().ToLowerInvariant();
|
||||
|
||||
return new RawSourceMetadata(
|
||||
vendor,
|
||||
connector,
|
||||
version,
|
||||
stream);
|
||||
}
|
||||
|
||||
private RawUpstreamMetadata CreateUpstreamMetadata(VexRawDocument document, ImmutableDictionary<string, string> metadata)
|
||||
{
|
||||
var upstreamId = TryMetadata(
|
||||
metadata,
|
||||
"upstream.id",
|
||||
"aoc.upstream_id",
|
||||
"vulnerability.id",
|
||||
"advisory.id",
|
||||
"msrc.vulnerabilityId",
|
||||
"msrc.advisoryId",
|
||||
"oracle.csaf.entryId",
|
||||
"ubuntu.advisoryId",
|
||||
"cisco.csaf.documentId",
|
||||
"rancher.vex.id") ?? document.SourceUri.ToString();
|
||||
|
||||
var documentVersion = TryMetadata(
|
||||
metadata,
|
||||
"upstream.version",
|
||||
"aoc.document_version",
|
||||
"msrc.lastModified",
|
||||
"msrc.releaseDate",
|
||||
"oracle.csaf.revision",
|
||||
"ubuntu.version",
|
||||
"ubuntu.lastModified",
|
||||
"cisco.csaf.revision") ?? document.RetrievedAt.ToString("O");
|
||||
|
||||
var signature = CreateSignatureMetadata(metadata);
|
||||
var provenance = metadata;
|
||||
|
||||
return new RawUpstreamMetadata(
|
||||
sourceVendor,
|
||||
upstreamId,
|
||||
documentVersion,
|
||||
document.RetrievedAt,
|
||||
document.Digest,
|
||||
signature,
|
||||
provenance);
|
||||
}
|
||||
contentHash,
|
||||
document.SourceUri.ToString());
|
||||
fetchActivity?.SetTag("providerId", document.ProviderId);
|
||||
fetchActivity?.SetTag("format", document.Format.ToString().ToLowerInvariant());
|
||||
|
||||
private static RawSignatureMetadata CreateSignatureMetadata(ImmutableDictionary<string, string> metadata)
|
||||
{
|
||||
if (!TryBool(metadata, out var present, "signature.present", "aoc.signature.present"))
|
||||
VexRawDocumentRecord? existing;
|
||||
try
|
||||
{
|
||||
return new RawSignatureMetadata(false);
|
||||
var filter = Builders<VexRawDocumentRecord>.Filter.Eq(x => x.Id, document.Digest);
|
||||
existing = await _collection
|
||||
.Find(sessionHandle, filter)
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
fetchActivity?.SetTag("result", existing is null ? "miss" : "hit");
|
||||
fetchActivity?.SetStatus(ActivityStatusCode.Ok);
|
||||
}
|
||||
catch
|
||||
{
|
||||
fetchActivity?.SetStatus(ActivityStatusCode.Error, "lookup-failed");
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (fetchWatch.IsRunning)
|
||||
{
|
||||
fetchWatch.Stop();
|
||||
}
|
||||
|
||||
IngestionTelemetry.RecordLatency(tenant, sourceVendor, IngestionTelemetry.PhaseFetch, fetchWatch.Elapsed);
|
||||
}
|
||||
|
||||
if (!present)
|
||||
var record = VexRawDocumentRecord.FromDomain(document, includeContent: useInline);
|
||||
record.GridFsObjectId = useInline ? null : newGridId;
|
||||
|
||||
var writeWatch = Stopwatch.StartNew();
|
||||
using var writeActivity = IngestionTelemetry.StartWriteActivity(
|
||||
tenant,
|
||||
sourceVendor,
|
||||
upstreamId,
|
||||
contentHash,
|
||||
VexMongoCollectionNames.Raw);
|
||||
string? writeResult = null;
|
||||
|
||||
try
|
||||
{
|
||||
return new RawSignatureMetadata(false);
|
||||
await _collection
|
||||
.ReplaceOneAsync(
|
||||
sessionHandle,
|
||||
Builders<VexRawDocumentRecord>.Filter.Eq(x => x.Id, document.Digest),
|
||||
record,
|
||||
new ReplaceOptions { IsUpsert = true },
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
writeResult = existing is null ? IngestionTelemetry.ResultOk : IngestionTelemetry.ResultNoop;
|
||||
writeActivity?.SetTag("result", writeResult);
|
||||
|
||||
if (existing?.GridFsObjectId is string oldGridId && !string.IsNullOrWhiteSpace(oldGridId))
|
||||
{
|
||||
if (useInline || !string.Equals(newGridId, oldGridId, StringComparison.Ordinal))
|
||||
{
|
||||
oldGridIdToDelete = oldGridId;
|
||||
}
|
||||
}
|
||||
|
||||
if (startedTransaction)
|
||||
{
|
||||
await sessionHandle.CommitTransactionAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
writeActivity?.SetStatus(ActivityStatusCode.Ok);
|
||||
}
|
||||
catch
|
||||
{
|
||||
if (startedTransaction && sessionHandle.IsInTransaction)
|
||||
{
|
||||
await sessionHandle.AbortTransactionAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (!useInline && !string.IsNullOrWhiteSpace(newGridId))
|
||||
{
|
||||
await DeleteFromGridFsAsync(newGridId, sessionHandle, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
writeActivity?.SetStatus(ActivityStatusCode.Error, "write-failed");
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (writeWatch.IsRunning)
|
||||
{
|
||||
writeWatch.Stop();
|
||||
}
|
||||
|
||||
IngestionTelemetry.RecordLatency(tenant, sourceVendor, IngestionTelemetry.PhaseWrite, writeWatch.Elapsed);
|
||||
|
||||
if (!string.IsNullOrEmpty(writeResult))
|
||||
{
|
||||
IngestionTelemetry.RecordWriteAttempt(tenant, sourceVendor, writeResult);
|
||||
}
|
||||
}
|
||||
|
||||
var format = TryMetadata(metadata, "signature.format", "aoc.signature.format");
|
||||
var keyId = TryMetadata(metadata, "signature.key_id", "signature.keyId", "aoc.signature.key_id");
|
||||
var signature = TryMetadata(metadata, "signature.sig", "signature.signature", "aoc.signature.sig");
|
||||
var digest = TryMetadata(metadata, "signature.digest", "aoc.signature.digest");
|
||||
var certificate = TryMetadata(metadata, "signature.certificate", "aoc.signature.certificate");
|
||||
|
||||
return new RawSignatureMetadata(true, format, keyId, signature, certificate, digest);
|
||||
if (!string.IsNullOrWhiteSpace(oldGridIdToDelete))
|
||||
{
|
||||
await DeleteFromGridFsAsync(oldGridIdToDelete!, sessionHandle, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private RawContentMetadata CreateContent(VexRawDocument document, ImmutableDictionary<string, string> metadata)
|
||||
public async ValueTask<VexRawDocument?> FindByDigestAsync(string digest, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
if (document.Content.IsEmpty)
|
||||
if (string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
throw new InvalidOperationException("Raw VEX document content cannot be empty when enforcing AOC guard.");
|
||||
throw new ArgumentException("Digest must be provided.", nameof(digest));
|
||||
}
|
||||
|
||||
var trimmed = digest.Trim();
|
||||
var filter = Builders<VexRawDocumentRecord>.Filter.Eq(x => x.Id, trimmed);
|
||||
var record = session is null
|
||||
? await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false)
|
||||
: await _collection.Find(session, filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (record is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(record.GridFsObjectId))
|
||||
{
|
||||
var handle = session ?? await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
|
||||
var bytes = await DownloadFromGridFsAsync(record.GridFsObjectId, handle, cancellationToken).ConfigureAwait(false);
|
||||
return record.ToDomain(new ReadOnlyMemory<byte>(bytes));
|
||||
}
|
||||
|
||||
return record.ToDomain();
|
||||
}
|
||||
|
||||
private async Task<string?> UploadToGridFsAsync(VexRawDocument document, IClientSessionHandle? session, CancellationToken cancellationToken)
|
||||
{
|
||||
using var stream = new MemoryStream(document.Content.ToArray(), writable: false);
|
||||
var metadata = new BsonDocument
|
||||
{
|
||||
{ "providerId", document.ProviderId },
|
||||
{ "format", document.Format.ToString().ToLowerInvariant() },
|
||||
{ "sourceUri", document.SourceUri.ToString() },
|
||||
{ "retrievedAt", document.RetrievedAt.UtcDateTime },
|
||||
};
|
||||
|
||||
var options = new GridFSUploadOptions { Metadata = metadata };
|
||||
var objectId = await _bucket
|
||||
.UploadFromStreamAsync(document.Digest, stream, options, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return objectId.ToString();
|
||||
}
|
||||
|
||||
private async Task DeleteFromGridFsAsync(string gridFsObjectId, IClientSessionHandle? session, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!ObjectId.TryParse(gridFsObjectId, out var objectId))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var payload = JsonDocument.Parse(document.Content.ToArray());
|
||||
var raw = payload.RootElement.Clone();
|
||||
var specVersion = TryMetadata(metadata, "content.spec_version", "csaf.version", "openvex.version");
|
||||
var encoding = TryMetadata(metadata, "content.encoding");
|
||||
|
||||
return new RawContentMetadata(
|
||||
document.Format.ToString(),
|
||||
specVersion,
|
||||
raw,
|
||||
encoding);
|
||||
await _bucket.DeleteAsync(objectId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
catch (GridFSFileNotFoundException)
|
||||
{
|
||||
throw new InvalidOperationException("Raw VEX document payload must be valid JSON for AOC guard enforcement.", ex);
|
||||
// file already removed by TTL or manual cleanup
|
||||
}
|
||||
}
|
||||
|
||||
private static RawLinkset CreateLinkset()
|
||||
=> new()
|
||||
{
|
||||
Aliases = ImmutableArray<string>.Empty,
|
||||
PackageUrls = ImmutableArray<string>.Empty,
|
||||
Cpes = ImmutableArray<string>.Empty,
|
||||
References = ImmutableArray<RawReference>.Empty,
|
||||
ReconciledFrom = ImmutableArray<string>.Empty,
|
||||
Notes = ImmutableDictionary<string, string>.Empty,
|
||||
};
|
||||
|
||||
private static string? TryMetadata(ImmutableDictionary<string, string> metadata, params string[] keys)
|
||||
private async Task<byte[]> DownloadFromGridFsAsync(string gridFsObjectId, IClientSessionHandle? session, CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var key in keys)
|
||||
if (!ObjectId.TryParse(gridFsObjectId, out var objectId))
|
||||
{
|
||||
if (metadata.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return value;
|
||||
}
|
||||
return Array.Empty<byte>();
|
||||
}
|
||||
|
||||
return null;
|
||||
return await _bucket.DownloadAsBytesAsync(objectId, null, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static bool TryBool(ImmutableDictionary<string, string> metadata, out bool value, params string[] keys)
|
||||
{
|
||||
foreach (var key in keys)
|
||||
{
|
||||
if (metadata.TryGetValue(key, out var text) && bool.TryParse(text, out value))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
value = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string ExtractVendor(string providerId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(providerId))
|
||||
{
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
var trimmed = providerId.Trim();
|
||||
var separatorIndex = trimmed.LastIndexOfAny(new[] { ':', '.' });
|
||||
if (separatorIndex >= 0 && separatorIndex < trimmed.Length - 1)
|
||||
{
|
||||
return trimmed[(separatorIndex + 1)..];
|
||||
}
|
||||
|
||||
return trimmed;
|
||||
}
|
||||
async ValueTask IVexRawDocumentSink.StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> await StoreAsync(document, cancellationToken, session: null).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
@@ -12,7 +12,8 @@
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
|
||||
<ProjectReference Include="..\..\..\__Libraries\StellaOps.Ingestion.Telemetry\StellaOps.Ingestion.Telemetry.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
using System;
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json;
|
||||
using StellaOps.Excititor.Core;
|
||||
using RawVexDocumentModel = StellaOps.Concelier.RawModels.VexRawDocument;
|
||||
using RawSourceMetadata = StellaOps.Concelier.RawModels.RawSourceMetadata;
|
||||
using RawUpstreamMetadata = StellaOps.Concelier.RawModels.RawUpstreamMetadata;
|
||||
using RawContentMetadata = StellaOps.Concelier.RawModels.RawContent;
|
||||
using RawSignatureMetadata = StellaOps.Concelier.RawModels.RawSignatureMetadata;
|
||||
using RawLinkset = StellaOps.Concelier.RawModels.RawLinkset;
|
||||
using RawReference = StellaOps.Concelier.RawModels.RawReference;
|
||||
using VexStatementSummaryModel = StellaOps.Concelier.RawModels.VexStatementSummary;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
/// <summary>
|
||||
/// Converts Excititor domain VEX documents into Aggregation-Only Contract raw payloads.
|
||||
/// </summary>
|
||||
public static class VexRawDocumentMapper
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
public static RawVexDocumentModel ToRawModel(VexRawDocument document, string defaultTenant)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
var metadata = document.Metadata ?? ImmutableDictionary<string, string>.Empty;
|
||||
var tenant = ResolveTenant(metadata, defaultTenant);
|
||||
var source = CreateSourceMetadata(document, metadata);
|
||||
var upstream = CreateUpstreamMetadata(document, metadata);
|
||||
var content = CreateContent(document, metadata);
|
||||
var linkset = CreateLinkset();
|
||||
ImmutableArray<VexStatementSummaryModel>? statements = null;
|
||||
return new RawVexDocumentModel(tenant, source, upstream, content, linkset, statements);
|
||||
}
|
||||
|
||||
private static string ResolveTenant(ImmutableDictionary<string, string> metadata, string fallback)
|
||||
{
|
||||
var tenant = TryMetadata(metadata, "tenant", "aoc.tenant");
|
||||
if (string.IsNullOrWhiteSpace(tenant))
|
||||
{
|
||||
return (fallback ?? "tenant-default").Trim().ToLowerInvariant();
|
||||
}
|
||||
|
||||
return tenant.Trim().ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static RawSourceMetadata CreateSourceMetadata(VexRawDocument document, ImmutableDictionary<string, string> metadata)
|
||||
{
|
||||
var vendor = TryMetadata(metadata, "source.vendor", "connector.vendor") ?? ExtractVendor(document.ProviderId);
|
||||
var connector = TryMetadata(metadata, "source.connector") ?? document.ProviderId;
|
||||
var version = TryMetadata(metadata, "source.connector_version", "connector.version") ?? GetAssemblyVersion();
|
||||
var stream = TryMetadata(metadata, "source.stream", "connector.stream") ?? document.Format.ToString().ToLowerInvariant();
|
||||
return new RawSourceMetadata(vendor, connector, version, stream);
|
||||
}
|
||||
|
||||
private static string GetAssemblyVersion()
|
||||
=> typeof(VexRawDocumentMapper).Assembly.GetName().Version?.ToString() ?? "0.0.0";
|
||||
|
||||
private static RawUpstreamMetadata CreateUpstreamMetadata(VexRawDocument document, ImmutableDictionary<string, string> metadata)
|
||||
{
|
||||
var upstreamId = TryMetadata(
|
||||
metadata,
|
||||
"upstream.id",
|
||||
"aoc.upstream_id",
|
||||
"vulnerability.id",
|
||||
"advisory.id",
|
||||
"msrc.vulnerabilityId",
|
||||
"msrc.advisoryId",
|
||||
"oracle.csaf.entryId",
|
||||
"ubuntu.advisoryId",
|
||||
"cisco.csaf.documentId",
|
||||
"rancher.vex.id") ?? document.SourceUri.ToString();
|
||||
|
||||
var documentVersion = TryMetadata(
|
||||
metadata,
|
||||
"upstream.version",
|
||||
"aoc.document_version",
|
||||
"msrc.lastModified",
|
||||
"msrc.releaseDate",
|
||||
"oracle.csaf.revision",
|
||||
"ubuntu.version",
|
||||
"ubuntu.lastModified",
|
||||
"cisco.csaf.revision") ?? document.RetrievedAt.ToString("O");
|
||||
|
||||
var signature = CreateSignatureMetadata(metadata);
|
||||
return new RawUpstreamMetadata(
|
||||
upstreamId,
|
||||
documentVersion,
|
||||
document.RetrievedAt,
|
||||
document.Digest,
|
||||
signature,
|
||||
metadata);
|
||||
}
|
||||
|
||||
private static RawSignatureMetadata CreateSignatureMetadata(ImmutableDictionary<string, string> metadata)
|
||||
{
|
||||
if (!TryBool(metadata, out var present, "signature.present", "aoc.signature.present"))
|
||||
{
|
||||
return new RawSignatureMetadata(false);
|
||||
}
|
||||
|
||||
if (!present)
|
||||
{
|
||||
return new RawSignatureMetadata(false);
|
||||
}
|
||||
|
||||
var format = TryMetadata(metadata, "signature.format", "aoc.signature.format");
|
||||
var keyId = TryMetadata(metadata, "signature.key_id", "signature.keyId", "aoc.signature.key_id");
|
||||
var signature = TryMetadata(metadata, "signature.sig", "signature.signature", "aoc.signature.sig");
|
||||
var digest = TryMetadata(metadata, "signature.digest", "aoc.signature.digest");
|
||||
var certificate = TryMetadata(metadata, "signature.certificate", "aoc.signature.certificate");
|
||||
return new RawSignatureMetadata(true, format, keyId, signature, certificate, digest);
|
||||
}
|
||||
|
||||
private static RawContentMetadata CreateContent(VexRawDocument document, ImmutableDictionary<string, string> metadata)
|
||||
{
|
||||
if (document.Content.IsEmpty)
|
||||
{
|
||||
throw new InvalidOperationException("Raw VEX document content cannot be empty when enforcing AOC guard.");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var payload = JsonDocument.Parse(document.Content.ToArray());
|
||||
var raw = payload.RootElement.Clone();
|
||||
var specVersion = TryMetadata(metadata, "content.spec_version", "csaf.version", "openvex.version");
|
||||
var encoding = TryMetadata(metadata, "content.encoding");
|
||||
return new RawContentMetadata(
|
||||
document.Format.ToString(),
|
||||
specVersion,
|
||||
raw,
|
||||
encoding);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new InvalidOperationException("Raw VEX document payload must be valid JSON for AOC guard enforcement.", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private static RawLinkset CreateLinkset()
|
||||
=> new()
|
||||
{
|
||||
Aliases = ImmutableArray<string>.Empty,
|
||||
PackageUrls = ImmutableArray<string>.Empty,
|
||||
Cpes = ImmutableArray<string>.Empty,
|
||||
References = ImmutableArray<RawReference>.Empty,
|
||||
ReconciledFrom = ImmutableArray<string>.Empty,
|
||||
Notes = ImmutableDictionary<string, string>.Empty,
|
||||
};
|
||||
|
||||
private static string? TryMetadata(ImmutableDictionary<string, string> metadata, params string[] keys)
|
||||
{
|
||||
foreach (var key in keys)
|
||||
{
|
||||
if (metadata.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool TryBool(ImmutableDictionary<string, string> metadata, out bool value, params string[] keys)
|
||||
{
|
||||
foreach (var key in keys)
|
||||
{
|
||||
if (metadata.TryGetValue(key, out var text) && bool.TryParse(text, out value))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
value = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string ExtractVendor(string providerId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(providerId))
|
||||
{
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
var trimmed = providerId.Trim();
|
||||
var separatorIndex = trimmed.LastIndexOfAny(new[] { ':', '.' });
|
||||
if (separatorIndex >= 0 && separatorIndex < trimmed.Length - 1)
|
||||
{
|
||||
return trimmed[(separatorIndex + 1)..];
|
||||
}
|
||||
|
||||
return trimmed;
|
||||
}
|
||||
}
|
||||
@@ -1,23 +1,20 @@
|
||||
using System.Collections.Generic;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Mongo2Go;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Tests;
|
||||
|
||||
public sealed class MongoVexCacheMaintenanceTests : IAsyncLifetime
|
||||
{
|
||||
private readonly MongoDbRunner _runner;
|
||||
private readonly IMongoDatabase _database;
|
||||
|
||||
public MongoVexCacheMaintenanceTests()
|
||||
{
|
||||
_runner = MongoDbRunner.Start();
|
||||
var client = new MongoClient(_runner.ConnectionString);
|
||||
_database = client.GetDatabase("vex-cache-maintenance-tests");
|
||||
VexMongoMappingRegistry.Register();
|
||||
}
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Tests;
|
||||
|
||||
public sealed class MongoVexCacheMaintenanceTests : IAsyncLifetime
|
||||
{
|
||||
private readonly TestMongoEnvironment _mongo = new();
|
||||
private readonly IMongoDatabase _database;
|
||||
|
||||
public MongoVexCacheMaintenanceTests()
|
||||
{
|
||||
_database = _mongo.CreateDatabase("cache-maintenance");
|
||||
VexMongoMappingRegistry.Register();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RemoveExpiredAsync_DeletesEntriesBeforeCutoff()
|
||||
@@ -114,9 +111,5 @@ public sealed class MongoVexCacheMaintenanceTests : IAsyncLifetime
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public Task DisposeAsync()
|
||||
{
|
||||
_runner.Dispose();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
public Task DisposeAsync() => _mongo.DisposeAsync();
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Mongo2Go;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Aoc;
|
||||
@@ -13,21 +12,20 @@ using RawVexDocumentModel = StellaOps.Concelier.RawModels.VexRawDocument;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Tests;
|
||||
|
||||
public sealed class MongoVexRepositoryTests : IAsyncLifetime
|
||||
{
|
||||
private readonly MongoDbRunner _runner;
|
||||
private readonly MongoClient _client;
|
||||
|
||||
public MongoVexRepositoryTests()
|
||||
{
|
||||
_runner = MongoDbRunner.Start();
|
||||
_client = new MongoClient(_runner.ConnectionString);
|
||||
}
|
||||
public sealed class MongoVexRepositoryTests : IAsyncLifetime
|
||||
{
|
||||
private readonly TestMongoEnvironment _mongo = new();
|
||||
private readonly MongoClient _client;
|
||||
|
||||
public MongoVexRepositoryTests()
|
||||
{
|
||||
_client = _mongo.Client;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RawStore_UsesGridFsForLargePayloads()
|
||||
{
|
||||
var database = _client.GetDatabase($"vex-raw-gridfs-{Guid.NewGuid():N}");
|
||||
var database = _mongo.CreateDatabase("vex-raw-gridfs");
|
||||
var store = CreateRawStore(database, thresholdBytes: 32);
|
||||
|
||||
var payload = CreateJsonPayload(new string('A', 256));
|
||||
@@ -63,7 +61,7 @@ public sealed class MongoVexRepositoryTests : IAsyncLifetime
|
||||
[Fact]
|
||||
public async Task RawStore_ReplacesGridFsWithInlinePayload()
|
||||
{
|
||||
var database = _client.GetDatabase($"vex-raw-inline-{Guid.NewGuid():N}");
|
||||
var database = _mongo.CreateDatabase("vex-raw-inline");
|
||||
var store = CreateRawStore(database, thresholdBytes: 16);
|
||||
|
||||
var largePayload = CreateJsonPayload(new string('B', 128));
|
||||
@@ -176,7 +174,7 @@ public sealed class MongoVexRepositoryTests : IAsyncLifetime
|
||||
[Fact]
|
||||
public async Task ExportStore_FindAsync_ExpiresCacheEntries()
|
||||
{
|
||||
var database = _client.GetDatabase($"vex-export-expire-{Guid.NewGuid():N}");
|
||||
var database = _mongo.CreateDatabase("vex-export-expire");
|
||||
var options = Options.Create(new VexMongoStorageOptions
|
||||
{
|
||||
ExportCacheTtl = TimeSpan.FromMinutes(5),
|
||||
@@ -217,7 +215,7 @@ public sealed class MongoVexRepositoryTests : IAsyncLifetime
|
||||
[Fact]
|
||||
public async Task ClaimStore_AppendsAndQueriesStatements()
|
||||
{
|
||||
var database = _client.GetDatabase($"vex-claims-{Guid.NewGuid():N}");
|
||||
var database = _mongo.CreateDatabase("vex-claims");
|
||||
var store = new MongoVexClaimStore(database);
|
||||
|
||||
var product = new VexProduct("pkg:demo/app", "Demo App", version: "1.0.0", purl: "pkg:demo/app@1.0.0");
|
||||
@@ -305,11 +303,7 @@ public sealed class MongoVexRepositoryTests : IAsyncLifetime
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public Task DisposeAsync()
|
||||
{
|
||||
_runner.Dispose();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
public Task DisposeAsync() => _mongo.DisposeAsync();
|
||||
|
||||
private static byte[] CreateJsonPayload(string value)
|
||||
=> Encoding.UTF8.GetBytes(CreateJsonPayloadString(value));
|
||||
|
||||
@@ -2,23 +2,23 @@ using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Mongo2Go;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Tests;
|
||||
|
||||
public sealed class MongoVexSessionConsistencyTests : IAsyncLifetime
|
||||
{
|
||||
private readonly MongoDbRunner _runner;
|
||||
|
||||
public MongoVexSessionConsistencyTests()
|
||||
{
|
||||
_runner = MongoDbRunner.Start(singleNodeReplSet: true);
|
||||
}
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Tests;
|
||||
|
||||
public sealed class MongoVexSessionConsistencyTests : IAsyncLifetime
|
||||
{
|
||||
private readonly TestMongoEnvironment _mongo = new();
|
||||
private readonly MongoClient _client;
|
||||
|
||||
public MongoVexSessionConsistencyTests()
|
||||
{
|
||||
_client = _mongo.Client;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionProvidesReadYourWrites()
|
||||
@@ -45,7 +45,7 @@ public sealed class MongoVexSessionConsistencyTests : IAsyncLifetime
|
||||
await using var provider = BuildServiceProvider();
|
||||
await using var scope = provider.CreateAsyncScope();
|
||||
|
||||
var client = scope.ServiceProvider.GetRequiredService<IMongoClient>();
|
||||
var client = scope.ServiceProvider.GetRequiredService<IMongoClient>();
|
||||
var sessionProvider = scope.ServiceProvider.GetRequiredService<IVexMongoSessionProvider>();
|
||||
var providerStore = scope.ServiceProvider.GetRequiredService<IVexProviderStore>();
|
||||
|
||||
@@ -74,18 +74,18 @@ public sealed class MongoVexSessionConsistencyTests : IAsyncLifetime
|
||||
|
||||
private ServiceProvider BuildServiceProvider()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging(builder => builder.AddDebug());
|
||||
services.Configure<VexMongoStorageOptions>(options =>
|
||||
{
|
||||
options.ConnectionString = _runner.ConnectionString;
|
||||
options.DatabaseName = $"excititor-session-tests-{Guid.NewGuid():N}";
|
||||
options.CommandTimeout = TimeSpan.FromSeconds(5);
|
||||
options.RawBucketName = "vex.raw";
|
||||
});
|
||||
services.AddExcititorMongoStorage();
|
||||
return services.BuildServiceProvider();
|
||||
}
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging(builder => builder.AddDebug());
|
||||
services.Configure<VexMongoStorageOptions>(options =>
|
||||
{
|
||||
options.ConnectionString = _mongo.ConnectionString;
|
||||
options.DatabaseName = _mongo.ReserveDatabase("session");
|
||||
options.CommandTimeout = TimeSpan.FromSeconds(5);
|
||||
options.RawBucketName = "vex.raw";
|
||||
});
|
||||
services.AddExcititorMongoStorage();
|
||||
return services.BuildServiceProvider();
|
||||
}
|
||||
|
||||
private static async Task ExecuteWithRetryAsync(Func<Task> action, CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -176,9 +176,5 @@ public sealed class MongoVexSessionConsistencyTests : IAsyncLifetime
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public Task DisposeAsync()
|
||||
{
|
||||
_runner.Dispose();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
public Task DisposeAsync() => _mongo.DisposeAsync();
|
||||
}
|
||||
|
||||
@@ -3,22 +3,22 @@ using System.Collections.Immutable;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Mongo2Go;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System.Text;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Aoc;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Tests;
|
||||
|
||||
public sealed class MongoVexStatementBackfillServiceTests : IAsyncLifetime
|
||||
{
|
||||
private readonly MongoDbRunner _runner;
|
||||
|
||||
public MongoVexStatementBackfillServiceTests()
|
||||
{
|
||||
_runner = MongoDbRunner.Start(singleNodeReplSet: true);
|
||||
}
|
||||
using RawVexDocumentModel = StellaOps.Concelier.RawModels.VexRawDocument;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Tests;
|
||||
|
||||
public sealed class MongoVexStatementBackfillServiceTests : IAsyncLifetime
|
||||
{
|
||||
private readonly TestMongoEnvironment _mongo = new();
|
||||
|
||||
public MongoVexStatementBackfillServiceTests()
|
||||
{
|
||||
// Intentionally left blank; Mongo environment is initialized on demand.
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RunAsync_BackfillsStatementsFromRawDocuments()
|
||||
@@ -108,34 +108,32 @@ public sealed class MongoVexStatementBackfillServiceTests : IAsyncLifetime
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging(builder => builder.AddDebug());
|
||||
services.AddSingleton(TimeProvider.System);
|
||||
services.Configure<VexMongoStorageOptions>(options =>
|
||||
{
|
||||
options.ConnectionString = _runner.ConnectionString;
|
||||
options.DatabaseName = $"excititor-backfill-tests-{Guid.NewGuid():N}";
|
||||
options.CommandTimeout = TimeSpan.FromSeconds(5);
|
||||
options.RawBucketName = "vex.raw";
|
||||
options.GridFsInlineThresholdBytes = 1024;
|
||||
options.ExportCacheTtl = TimeSpan.FromHours(1);
|
||||
});
|
||||
services.Configure<VexMongoStorageOptions>(options =>
|
||||
{
|
||||
options.ConnectionString = _mongo.ConnectionString;
|
||||
options.DatabaseName = _mongo.ReserveDatabase("backfill");
|
||||
options.CommandTimeout = TimeSpan.FromSeconds(5);
|
||||
options.RawBucketName = "vex.raw";
|
||||
options.GridFsInlineThresholdBytes = 1024;
|
||||
options.ExportCacheTtl = TimeSpan.FromHours(1);
|
||||
options.DefaultTenant = "tests";
|
||||
});
|
||||
services.AddExcititorMongoStorage();
|
||||
services.AddExcititorAocGuards();
|
||||
services.AddSingleton<IVexRawWriteGuard, PermissiveVexRawWriteGuard>();
|
||||
services.AddSingleton<IVexNormalizer, TestNormalizer>();
|
||||
return services.BuildServiceProvider();
|
||||
}
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public Task DisposeAsync()
|
||||
{
|
||||
_runner.Dispose();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
public Task DisposeAsync() => _mongo.DisposeAsync();
|
||||
|
||||
private static ReadOnlyMemory<byte> CreateJsonPayload(string value)
|
||||
=> Encoding.UTF8.GetBytes($"{{\"data\":\"{value}\"}}");
|
||||
|
||||
private sealed class TestNormalizer : IVexNormalizer
|
||||
{
|
||||
private sealed class TestNormalizer : IVexNormalizer
|
||||
{
|
||||
public string Format => "csaf";
|
||||
|
||||
public bool CanHandle(VexRawDocument document) => true;
|
||||
@@ -171,6 +169,14 @@ public sealed class MongoVexStatementBackfillServiceTests : IAsyncLifetime
|
||||
|
||||
var claims = ImmutableArray.Create(claim);
|
||||
return ValueTask.FromResult(new VexClaimBatch(document, claims, ImmutableDictionary<string, string>.Empty));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class PermissiveVexRawWriteGuard : IVexRawWriteGuard
|
||||
{
|
||||
public void EnsureValid(RawVexDocumentModel document)
|
||||
{
|
||||
// Tests control the payloads; guard bypass keeps focus on backfill logic.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,23 +1,20 @@
|
||||
using System.Globalization;
|
||||
using Mongo2Go;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Tests;
|
||||
|
||||
public sealed class MongoVexStoreMappingTests : IAsyncLifetime
|
||||
{
|
||||
private readonly MongoDbRunner _runner;
|
||||
private readonly IMongoDatabase _database;
|
||||
|
||||
public MongoVexStoreMappingTests()
|
||||
{
|
||||
_runner = MongoDbRunner.Start();
|
||||
var client = new MongoClient(_runner.ConnectionString);
|
||||
_database = client.GetDatabase("excititor-storage-mapping-tests");
|
||||
VexMongoMappingRegistry.Register();
|
||||
}
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Tests;
|
||||
|
||||
public sealed class MongoVexStoreMappingTests : IAsyncLifetime
|
||||
{
|
||||
private readonly TestMongoEnvironment _mongo = new();
|
||||
private readonly IMongoDatabase _database;
|
||||
|
||||
public MongoVexStoreMappingTests()
|
||||
{
|
||||
_database = _mongo.CreateDatabase("storage-mapping");
|
||||
VexMongoMappingRegistry.Register();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProviderStore_RoundTrips_WithExtraFields()
|
||||
@@ -259,9 +256,5 @@ public sealed class MongoVexStoreMappingTests : IAsyncLifetime
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public Task DisposeAsync()
|
||||
{
|
||||
_runner.Dispose();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
public Task DisposeAsync() => _mongo.DisposeAsync();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Mongo2Go;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Tests;
|
||||
|
||||
internal sealed class TestMongoEnvironment : IAsyncLifetime
|
||||
{
|
||||
private const string Prefix = "exstor";
|
||||
private readonly MongoDbRunner? _runner;
|
||||
private readonly HashSet<string> _reservedDatabases = new(StringComparer.Ordinal);
|
||||
|
||||
public TestMongoEnvironment()
|
||||
{
|
||||
var overrideConnection = Environment.GetEnvironmentVariable("EXCITITOR_TEST_MONGO_URI");
|
||||
if (!string.IsNullOrWhiteSpace(overrideConnection))
|
||||
{
|
||||
ConnectionString = overrideConnection.Trim();
|
||||
Client = new MongoClient(ConnectionString);
|
||||
return;
|
||||
}
|
||||
|
||||
_runner = MongoDbRunner.Start(singleNodeReplSet: true);
|
||||
ConnectionString = _runner.ConnectionString;
|
||||
Client = new MongoClient(ConnectionString);
|
||||
}
|
||||
|
||||
public MongoClient Client { get; }
|
||||
|
||||
public string ConnectionString { get; }
|
||||
|
||||
public string ReserveDatabase(string hint)
|
||||
{
|
||||
var baseName = string.IsNullOrWhiteSpace(hint) ? "db" : hint.ToLowerInvariant();
|
||||
var builder = new StringBuilder(baseName.Length);
|
||||
foreach (var ch in baseName)
|
||||
{
|
||||
builder.Append(char.IsLetterOrDigit(ch) ? ch : '_');
|
||||
}
|
||||
|
||||
var slug = builder.Length == 0 ? "db" : builder.ToString();
|
||||
var suffix = ObjectId.GenerateNewId().ToString();
|
||||
var maxSlugLength = Math.Max(1, 60 - Prefix.Length - suffix.Length - 2);
|
||||
if (slug.Length > maxSlugLength)
|
||||
{
|
||||
slug = slug[..maxSlugLength];
|
||||
}
|
||||
|
||||
var name = $"{Prefix}_{slug}_{suffix}";
|
||||
_reservedDatabases.Add(name);
|
||||
return name;
|
||||
}
|
||||
|
||||
public IMongoDatabase CreateDatabase(string hint)
|
||||
{
|
||||
var name = ReserveDatabase(hint);
|
||||
return Client.GetDatabase(name);
|
||||
}
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
if (_runner is not null)
|
||||
{
|
||||
_runner.Dispose();
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var db in _reservedDatabases)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Client.DropDatabaseAsync(db);
|
||||
}
|
||||
catch (MongoException)
|
||||
{
|
||||
// best-effort cleanup when sharing a developer-managed instance.
|
||||
}
|
||||
}
|
||||
|
||||
_reservedDatabases.Clear();
|
||||
}
|
||||
}
|
||||
@@ -1,25 +1,22 @@
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Mongo2Go;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Storage.Mongo.Migrations;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Storage.Mongo.Migrations;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Tests;
|
||||
|
||||
public sealed class VexMongoMigrationRunnerTests : IAsyncLifetime
|
||||
{
|
||||
private readonly MongoDbRunner _runner;
|
||||
private readonly IMongoDatabase _database;
|
||||
|
||||
public VexMongoMigrationRunnerTests()
|
||||
{
|
||||
_runner = MongoDbRunner.Start();
|
||||
var client = new MongoClient(_runner.ConnectionString);
|
||||
_database = client.GetDatabase("excititor-migrations-tests");
|
||||
}
|
||||
public sealed class VexMongoMigrationRunnerTests : IAsyncLifetime
|
||||
{
|
||||
private readonly TestMongoEnvironment _mongo = new();
|
||||
private readonly IMongoDatabase _database;
|
||||
|
||||
public VexMongoMigrationRunnerTests()
|
||||
{
|
||||
_database = _mongo.CreateDatabase("migrations");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RunAsync_AppliesInitialIndexesOnce()
|
||||
@@ -60,9 +57,5 @@ public sealed class VexMongoMigrationRunnerTests : IAsyncLifetime
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public Task DisposeAsync()
|
||||
{
|
||||
_runner.Dispose();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
public Task DisposeAsync() => _mongo.DisposeAsync();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,451 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.Metrics;
|
||||
using System.Linq;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Net.Http.Json;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using EphemeralMongo;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Concelier.RawModels;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using StellaOps.Ingestion.Telemetry;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Tests;
|
||||
|
||||
public sealed class BatchIngestValidationTests : IDisposable
|
||||
{
|
||||
private const string Tenant = "tests";
|
||||
|
||||
private readonly IMongoRunner _runner;
|
||||
private readonly TestWebApplicationFactory _factory;
|
||||
|
||||
public BatchIngestValidationTests()
|
||||
{
|
||||
_runner = MongoRunner.Run(new MongoRunnerOptions { UseSingleNodeReplicaSet = true });
|
||||
|
||||
_factory = new TestWebApplicationFactory(
|
||||
configureConfiguration: configuration =>
|
||||
{
|
||||
configuration.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["Excititor:Storage:Mongo:ConnectionString"] = _runner.ConnectionString,
|
||||
["Excititor:Storage:Mongo:DatabaseName"] = "vex_batch_tests",
|
||||
["Excititor:Storage:Mongo:DefaultTenant"] = Tenant,
|
||||
});
|
||||
},
|
||||
configureServices: services =>
|
||||
{
|
||||
TestServiceOverrides.Apply(services);
|
||||
services.AddTestAuthentication();
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", "BatchIngestValidation")]
|
||||
public async Task BatchFixturesMaintainParityMetricsAndVerify()
|
||||
{
|
||||
using var metrics = new IngestionMetricListener();
|
||||
using var client = CreateClient();
|
||||
|
||||
var fixtures = VexFixtureLibrary.CreateBatch();
|
||||
foreach (var fixture in fixtures)
|
||||
{
|
||||
var ingestResponse = await client.PostAsJsonAsync("/ingest/vex", fixture.Request);
|
||||
ingestResponse.EnsureSuccessStatusCode();
|
||||
var payload = await ingestResponse.Content.ReadFromJsonAsync<VexIngestResponse>();
|
||||
Assert.NotNull(payload);
|
||||
fixture.RecordDigest(payload!.Digest);
|
||||
}
|
||||
|
||||
var listResponse = await client.GetAsync($"/vex/raw?limit={fixtures.Count * 2}");
|
||||
listResponse.EnsureSuccessStatusCode();
|
||||
var listPayload = await listResponse.Content.ReadFromJsonAsync<VexRawListResponse>();
|
||||
Assert.NotNull(listPayload);
|
||||
foreach (var fixture in fixtures)
|
||||
{
|
||||
Assert.Contains(listPayload!.Records, record => record.Digest == fixture.Digest);
|
||||
}
|
||||
|
||||
foreach (var fixture in fixtures)
|
||||
{
|
||||
var recordResponse = await client.GetAsync($"/vex/raw/{Uri.EscapeDataString(fixture.Digest)}");
|
||||
recordResponse.EnsureSuccessStatusCode();
|
||||
var record = await recordResponse.Content.ReadFromJsonAsync<VexRawRecordResponse>();
|
||||
Assert.NotNull(record);
|
||||
fixture.AssertRecord(record!);
|
||||
}
|
||||
|
||||
var verifyRequest = new VexAocVerifyRequest(
|
||||
DateTimeOffset.UtcNow.AddMinutes(-5),
|
||||
DateTimeOffset.UtcNow.AddMinutes(5),
|
||||
fixtures.Count + 5,
|
||||
null,
|
||||
null);
|
||||
var verifyResponse = await client.PostAsJsonAsync("/aoc/verify", verifyRequest);
|
||||
verifyResponse.EnsureSuccessStatusCode();
|
||||
var verifyPayload = await verifyResponse.Content.ReadFromJsonAsync<VexAocVerifyResponse>();
|
||||
Assert.NotNull(verifyPayload);
|
||||
Assert.Equal(Tenant, verifyPayload!.Tenant);
|
||||
Assert.Equal(fixtures.Count, verifyPayload.Checked.Vex);
|
||||
Assert.Empty(verifyPayload.Violations);
|
||||
Assert.Equal(fixtures.Count, verifyPayload.Metrics.IngestionWriteTotal);
|
||||
Assert.Equal(0, verifyPayload.Metrics.AocViolationTotal);
|
||||
Assert.False(verifyPayload.Truncated);
|
||||
|
||||
Assert.True(metrics.WaitForMeasurements(fixtures.Count, TimeSpan.FromSeconds(2)));
|
||||
foreach (var measurement in metrics.GetMeasurements())
|
||||
{
|
||||
Assert.Equal(Tenant, measurement.Tenant);
|
||||
Assert.Equal(IngestionTelemetry.ResultOk, measurement.Result);
|
||||
Assert.Equal(1, measurement.Value);
|
||||
}
|
||||
}
|
||||
|
||||
private HttpClient CreateClient()
|
||||
{
|
||||
var client = _factory.CreateClient(new WebApplicationFactoryClientOptions
|
||||
{
|
||||
AllowAutoRedirect = false
|
||||
});
|
||||
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "vex.admin vex.read");
|
||||
client.DefaultRequestHeaders.Add("X-Stella-Tenant", Tenant);
|
||||
return client;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_factory.Dispose();
|
||||
_runner.Dispose();
|
||||
}
|
||||
|
||||
private sealed class IngestionMetricListener : IDisposable
|
||||
{
|
||||
private readonly List<Measurement> _measurements = new();
|
||||
private readonly MeterListener _listener;
|
||||
|
||||
public IngestionMetricListener()
|
||||
{
|
||||
_listener = new MeterListener
|
||||
{
|
||||
InstrumentPublished = (instrument, listener) =>
|
||||
{
|
||||
if (instrument.Meter.Name == IngestionTelemetry.MeterName &&
|
||||
instrument.Name == "ingestion_write_total")
|
||||
{
|
||||
listener.EnableMeasurementEvents(instrument);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_listener.SetMeasurementEventCallback<long>((instrument, measurement, tags, state) =>
|
||||
{
|
||||
if (instrument.Meter.Name != IngestionTelemetry.MeterName ||
|
||||
instrument.Name != "ingestion_write_total")
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
string tenant = string.Empty;
|
||||
string source = string.Empty;
|
||||
string result = string.Empty;
|
||||
|
||||
foreach (var tag in tags)
|
||||
{
|
||||
switch (tag.Key)
|
||||
{
|
||||
case "tenant":
|
||||
tenant = tag.Value?.ToString() ?? string.Empty;
|
||||
break;
|
||||
case "source":
|
||||
source = tag.Value?.ToString() ?? string.Empty;
|
||||
break;
|
||||
case "result":
|
||||
result = tag.Value?.ToString() ?? string.Empty;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
lock (_measurements)
|
||||
{
|
||||
_measurements.Add(new Measurement(tenant, source, result, measurement));
|
||||
}
|
||||
});
|
||||
|
||||
_listener.Start();
|
||||
}
|
||||
|
||||
public bool WaitForMeasurements(int expected, TimeSpan timeout)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
while (sw.Elapsed < timeout)
|
||||
{
|
||||
lock (_measurements)
|
||||
{
|
||||
if (_measurements.Count >= expected)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
Thread.Sleep(25);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public IReadOnlyList<Measurement> GetMeasurements()
|
||||
{
|
||||
lock (_measurements)
|
||||
{
|
||||
return _measurements.ToList();
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose() => _listener.Dispose();
|
||||
|
||||
internal sealed record Measurement(string Tenant, string Source, string Result, long Value);
|
||||
}
|
||||
|
||||
private sealed record VexFixture(
|
||||
string Name,
|
||||
VexIngestRequest Request,
|
||||
string ExpectedFormat,
|
||||
Action<JsonElement> ContentAssertion)
|
||||
{
|
||||
private string? _digest;
|
||||
|
||||
public string Digest => _digest ?? throw new InvalidOperationException("Digest not recorded yet.");
|
||||
|
||||
public void RecordDigest(string digest)
|
||||
{
|
||||
_digest = digest ?? throw new ArgumentNullException(nameof(digest));
|
||||
}
|
||||
|
||||
public void AssertRecord(VexRawRecordResponse record)
|
||||
{
|
||||
Assert.Equal(ExpectedFormat, record.Document.Content.Format, StringComparer.OrdinalIgnoreCase);
|
||||
ContentAssertion(record.Document.Content.Raw);
|
||||
}
|
||||
}
|
||||
|
||||
private static class VexFixtureLibrary
|
||||
{
|
||||
public static IReadOnlyList<VexFixture> CreateBatch()
|
||||
=> new[]
|
||||
{
|
||||
CreateCycloneDxFixture("001", "sha256:batch-cdx-001", "CDX-BATCH-001", "not_affected"),
|
||||
CreateCycloneDxFixture("002", "sha256:batch-cdx-002", "CDX-BATCH-002", "affected"),
|
||||
CreateCycloneDxFixture("003", "sha256:batch-cdx-003", "CDX-BATCH-003", "fixed"),
|
||||
CreateCsafFixture("010", "sha256:batch-csaf-001", "CSAF-BATCH-001", "fixed"),
|
||||
CreateCsafFixture("011", "sha256:batch-csaf-002", "CSAF-BATCH-002", "known_affected"),
|
||||
CreateCsafFixture("012", "sha256:batch-csaf-003", "CSAF-BATCH-003", "known_not_affected"),
|
||||
CreateOpenVexFixture("020", "sha256:batch-openvex-001", "OVX-BATCH-001", "affected"),
|
||||
CreateOpenVexFixture("021", "sha256:batch-openvex-002", "OVX-BATCH-002", "not_affected"),
|
||||
CreateOpenVexFixture("022", "sha256:batch-openvex-003", "OVX-BATCH-003", "fixed")
|
||||
};
|
||||
|
||||
private static VexFixture CreateCycloneDxFixture(string suffix, string digest, string upstreamId, string state)
|
||||
{
|
||||
var vulnerabilityId = $"CVE-2025-{suffix}";
|
||||
var raw = $$"""
|
||||
{
|
||||
"bomFormat": "CycloneDX",
|
||||
"specVersion": "1.6",
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"timestamp": "2025-11-08T00:00:00Z",
|
||||
"tools": [
|
||||
{ "vendor": "stellaops", "name": "batch-cdx" }
|
||||
]
|
||||
},
|
||||
"vulnerabilities": [
|
||||
{
|
||||
"id": "{{vulnerabilityId}}",
|
||||
"analysis": { "state": "{{state}}" },
|
||||
"ratings": [
|
||||
{ "score": 0.0, "method": "cvssv3" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
""";
|
||||
|
||||
return new VexFixture(
|
||||
$"cyclonedx-{suffix}",
|
||||
BuildRequest(
|
||||
providerId: $"cyclonedx:batch:{suffix}",
|
||||
vendor: "vendor:cyclonedx",
|
||||
connector: "cdx-batch",
|
||||
stream: "cyclonedx-vex",
|
||||
format: "cyclonedx",
|
||||
specVersion: "1.6",
|
||||
rawJson: raw,
|
||||
digest: digest,
|
||||
upstreamId: upstreamId,
|
||||
sourceUri: $"https://example.test/vex/cyclonedx/{suffix}"),
|
||||
"cyclonedx",
|
||||
element =>
|
||||
{
|
||||
var actual = element
|
||||
.GetProperty("vulnerabilities")[0]
|
||||
.GetProperty("analysis")
|
||||
.GetProperty("state")
|
||||
.GetString();
|
||||
Assert.Equal(state, actual);
|
||||
});
|
||||
}
|
||||
|
||||
private static VexFixture CreateCsafFixture(string suffix, string digest, string upstreamId, string statusKey)
|
||||
{
|
||||
var cve = $"CVE-2025-{suffix}";
|
||||
var productId = $"csaf-prod-{suffix}";
|
||||
var raw = $$"""
|
||||
{
|
||||
"document": {
|
||||
"category": "csaf_vex",
|
||||
"title": "Sample CSAF VEX",
|
||||
"tracking": {
|
||||
"id": "CSAF-2025-{{suffix}}",
|
||||
"version": "1",
|
||||
"current_release_date": "2025-11-07T00:00:00Z",
|
||||
"initial_release_date": "2025-11-07T00:00:00Z",
|
||||
"status": "final"
|
||||
}
|
||||
},
|
||||
"product_tree": {
|
||||
"branches": [
|
||||
{
|
||||
"name": "products",
|
||||
"product": {
|
||||
"name": "sample-product-{{suffix}}",
|
||||
"product_id": "{{productId}}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"vulnerabilities": [
|
||||
{
|
||||
"cve": "{{cve}}",
|
||||
"product_status": {
|
||||
"{{statusKey}}": [ "{{productId}}" ]
|
||||
},
|
||||
"threats": [
|
||||
{ "category": "impact", "details": "none" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
""";
|
||||
|
||||
return new VexFixture(
|
||||
$"csaf-{suffix}",
|
||||
BuildRequest(
|
||||
providerId: $"csaf:batch:{suffix}",
|
||||
vendor: "vendor:csaf",
|
||||
connector: "csaf-batch",
|
||||
stream: "csaf-vex",
|
||||
format: "csaf",
|
||||
specVersion: "2.0",
|
||||
rawJson: raw,
|
||||
digest: digest,
|
||||
upstreamId: upstreamId,
|
||||
sourceUri: $"https://example.test/vex/csaf/{suffix}"),
|
||||
"csaf",
|
||||
element =>
|
||||
{
|
||||
var productStatus = element
|
||||
.GetProperty("vulnerabilities")[0]
|
||||
.GetProperty("product_status")
|
||||
.GetProperty(statusKey)
|
||||
.EnumerateArray()
|
||||
.First()
|
||||
.GetString();
|
||||
Assert.Equal(productId, productStatus);
|
||||
});
|
||||
}
|
||||
|
||||
private static VexFixture CreateOpenVexFixture(string suffix, string digest, string upstreamId, string status)
|
||||
{
|
||||
var raw = $$"""
|
||||
{
|
||||
"context": "https://openvex.dev/ns/v0.2.0",
|
||||
"statements": [
|
||||
{
|
||||
"vulnerability": "CVE-2025-{{suffix}}",
|
||||
"products": [
|
||||
"pkg:docker/demo@sha256:{{digest}}"
|
||||
],
|
||||
"status": "{{status}}",
|
||||
"statusNotes": "waiting on vendor patch"
|
||||
}
|
||||
]
|
||||
}
|
||||
""";
|
||||
|
||||
return new VexFixture(
|
||||
$"openvex-{suffix}",
|
||||
BuildRequest(
|
||||
providerId: $"openvex:batch:{suffix}",
|
||||
vendor: "vendor:openvex",
|
||||
connector: "openvex-batch",
|
||||
stream: "openvex",
|
||||
format: "openvex",
|
||||
specVersion: "0.2.0",
|
||||
rawJson: raw,
|
||||
digest: digest,
|
||||
upstreamId: upstreamId,
|
||||
sourceUri: $"https://example.test/vex/openvex/{suffix}"),
|
||||
"openvex",
|
||||
element =>
|
||||
{
|
||||
var actual = element
|
||||
.GetProperty("statements")[0]
|
||||
.GetProperty("status")
|
||||
.GetString();
|
||||
Assert.Equal(status, actual);
|
||||
});
|
||||
}
|
||||
|
||||
private static VexIngestRequest BuildRequest(
|
||||
string providerId,
|
||||
string vendor,
|
||||
string connector,
|
||||
string stream,
|
||||
string format,
|
||||
string specVersion,
|
||||
string rawJson,
|
||||
string digest,
|
||||
string upstreamId,
|
||||
string sourceUri)
|
||||
{
|
||||
using var rawDocument = JsonDocument.Parse(rawJson);
|
||||
var metadata = new Dictionary<string, string>
|
||||
{
|
||||
["source.vendor"] = vendor,
|
||||
["source.connector"] = connector,
|
||||
["source.stream"] = stream,
|
||||
["source.connector_version"] = "1.0.0"
|
||||
};
|
||||
|
||||
return new VexIngestRequest(
|
||||
providerId,
|
||||
new VexIngestSourceRequest(vendor, connector, "1.0.0", stream),
|
||||
new VexIngestUpstreamRequest(
|
||||
sourceUri,
|
||||
upstreamId,
|
||||
"1",
|
||||
DateTimeOffset.UtcNow,
|
||||
digest,
|
||||
new VexIngestSignatureRequest(false, null, null, null, null, null),
|
||||
new Dictionary<string, string>()),
|
||||
new VexIngestContentRequest(format, specVersion, rawDocument.RootElement.Clone(), null),
|
||||
metadata);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using EphemeralMongo;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using Xunit;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Tests;
|
||||
|
||||
public sealed class ObservabilityEndpointTests : IDisposable
|
||||
{
|
||||
private readonly TestWebApplicationFactory _factory;
|
||||
private readonly IMongoRunner _runner;
|
||||
|
||||
public ObservabilityEndpointTests()
|
||||
{
|
||||
_runner = MongoRunner.Run(new MongoRunnerOptions { UseSingleNodeReplicaSet = true });
|
||||
|
||||
_factory = new TestWebApplicationFactory(
|
||||
configureConfiguration: configuration =>
|
||||
{
|
||||
configuration.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["Excititor:Storage:Mongo:ConnectionString"] = _runner.ConnectionString,
|
||||
["Excititor:Storage:Mongo:DatabaseName"] = "excititor_obs_tests",
|
||||
["Excititor:Storage:Mongo:RawBucketName"] = "vex.raw",
|
||||
["Excititor:Observability:IngestWarningThreshold"] = "00:10:00",
|
||||
["Excititor:Observability:IngestCriticalThreshold"] = "00:30:00",
|
||||
["Excititor:Observability:SignatureWindow"] = "00:30:00",
|
||||
["Excititor:Observability:ConflictTrendWindow"] = "01:00:00",
|
||||
["Excititor:Observability:ConflictTrendBucketMinutes"] = "5"
|
||||
});
|
||||
},
|
||||
configureServices: services =>
|
||||
{
|
||||
TestServiceOverrides.Apply(services);
|
||||
services.AddTestAuthentication();
|
||||
|
||||
services.RemoveAll<IVexConnectorStateRepository>();
|
||||
services.AddScoped<IVexConnectorStateRepository, MongoVexConnectorStateRepository>();
|
||||
services.AddSingleton<IVexConnector>(_ => new StubConnector("excititor:redhat", VexProviderKind.Distro));
|
||||
});
|
||||
|
||||
SeedDatabase();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HealthEndpoint_ReturnsAggregatedMetrics()
|
||||
{
|
||||
var client = _factory.CreateClient(new WebApplicationFactoryClientOptions
|
||||
{
|
||||
AllowAutoRedirect = false
|
||||
});
|
||||
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "vex.admin");
|
||||
|
||||
using var response = await client.GetAsync("/obs/excititor/health");
|
||||
var payload = await response.Content.ReadAsStringAsync();
|
||||
Assert.True(response.IsSuccessStatusCode, payload);
|
||||
|
||||
using var document = JsonDocument.Parse(payload);
|
||||
var root = document.RootElement;
|
||||
|
||||
var ingest = root.GetProperty("ingest");
|
||||
Assert.Equal("healthy", ingest.GetProperty("status").GetString());
|
||||
|
||||
var connectors = ingest.GetProperty("connectors");
|
||||
Assert.Equal(1, connectors.GetArrayLength());
|
||||
Assert.Equal("excititor:redhat", connectors[0].GetProperty("connectorId").GetString());
|
||||
|
||||
var signature = root.GetProperty("signature");
|
||||
Assert.Equal(3, signature.GetProperty("documentsEvaluated").GetInt32());
|
||||
Assert.Equal(1, signature.GetProperty("failures").GetInt32());
|
||||
Assert.Equal(1, signature.GetProperty("verified").GetInt32());
|
||||
|
||||
var conflicts = root.GetProperty("conflicts");
|
||||
Assert.True(conflicts.GetProperty("conflictStatements").GetInt64() >= 2);
|
||||
Assert.True(conflicts.GetProperty("trend").GetArrayLength() >= 1);
|
||||
}
|
||||
|
||||
private void SeedDatabase()
|
||||
{
|
||||
using var scope = _factory.Services.CreateScope();
|
||||
var database = scope.ServiceProvider.GetRequiredService<IMongoDatabase>();
|
||||
database.DropCollection(VexMongoCollectionNames.Raw);
|
||||
database.DropCollection(VexMongoCollectionNames.Consensus);
|
||||
database.DropCollection(VexMongoCollectionNames.ConnectorState);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var rawCollection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
|
||||
rawCollection.InsertMany(new[]
|
||||
{
|
||||
new BsonDocument
|
||||
{
|
||||
{ "Id", "raw-1" },
|
||||
{ "ProviderId", "excititor:redhat" },
|
||||
{ ObservabilityEndpointTestsHelper.RetrievedAtField, now },
|
||||
{ ObservabilityEndpointTestsHelper.MetadataField, new BsonDocument { { "signature.present", "true" }, { "signature.verified", "true" } } }
|
||||
},
|
||||
new BsonDocument
|
||||
{
|
||||
{ "Id", "raw-2" },
|
||||
{ "ProviderId", "excititor:redhat" },
|
||||
{ ObservabilityEndpointTestsHelper.RetrievedAtField, now },
|
||||
{ ObservabilityEndpointTestsHelper.MetadataField, new BsonDocument { { "signature.present", "true" } } }
|
||||
},
|
||||
new BsonDocument
|
||||
{
|
||||
{ "Id", "raw-3" },
|
||||
{ "ProviderId", "excititor:redhat" },
|
||||
{ ObservabilityEndpointTestsHelper.RetrievedAtField, now },
|
||||
{ ObservabilityEndpointTestsHelper.MetadataField, new BsonDocument() }
|
||||
}
|
||||
});
|
||||
|
||||
var consensus = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Consensus);
|
||||
consensus.InsertMany(new[]
|
||||
{
|
||||
ObservabilityEndpointTestsHelper.CreateConsensusDocument("c1", now, "affected"),
|
||||
ObservabilityEndpointTestsHelper.CreateConsensusDocument("c2", now.AddMinutes(-5), "not_affected")
|
||||
});
|
||||
|
||||
var stateRepository = scope.ServiceProvider.GetRequiredService<IVexConnectorStateRepository>();
|
||||
var state = new VexConnectorState(
|
||||
"excititor:redhat",
|
||||
now.AddMinutes(-5),
|
||||
ImmutableArray<string>.Empty,
|
||||
ImmutableDictionary<string, string>.Empty,
|
||||
now.AddMinutes(-5),
|
||||
0,
|
||||
now.AddMinutes(10),
|
||||
null);
|
||||
stateRepository.SaveAsync(state, CancellationToken.None).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_factory.Dispose();
|
||||
_runner.Dispose();
|
||||
}
|
||||
|
||||
private sealed class StubConnector : IVexConnector
|
||||
{
|
||||
public StubConnector(string id, VexProviderKind kind)
|
||||
{
|
||||
Id = id;
|
||||
Kind = kind;
|
||||
}
|
||||
|
||||
public string Id { get; }
|
||||
|
||||
public VexProviderKind Kind { get; }
|
||||
|
||||
public ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, CancellationToken cancellationToken)
|
||||
=> AsyncEnumerable.Empty<VexRawDocument>();
|
||||
|
||||
public ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(new VexClaimBatch(
|
||||
document,
|
||||
ImmutableArray<VexClaim>.Empty,
|
||||
ImmutableDictionary<string, string>.Empty));
|
||||
}
|
||||
}
|
||||
|
||||
internal static class ObservabilityEndpointTestsHelper
|
||||
{
|
||||
public const string RetrievedAtField = "RetrievedAt";
|
||||
public const string MetadataField = "Metadata";
|
||||
|
||||
public static BsonDocument CreateConsensusDocument(string id, DateTime timestamp, string conflictStatus)
|
||||
{
|
||||
var conflicts = new BsonArray
|
||||
{
|
||||
new BsonDocument
|
||||
{
|
||||
{ "ProviderId", "excititor:redhat" },
|
||||
{ "Status", conflictStatus },
|
||||
{ "DocumentDigest", Guid.NewGuid().ToString("n") }
|
||||
}
|
||||
};
|
||||
|
||||
return new BsonDocument
|
||||
{
|
||||
{ "Id", id },
|
||||
{ "VulnerabilityId", $"CVE-{id}" },
|
||||
{ "Product", new BsonDocument { { "Key", $"pkg:{id}" }, { "Name", $"pkg-{id}" } } },
|
||||
{ "Status", "affected" },
|
||||
{ "CalculatedAt", timestamp },
|
||||
{ "Conflicts", conflicts }
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -19,8 +19,9 @@
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj" />
|
||||
<ProjectReference Include="../../../Aoc/__Libraries/StellaOps.Aoc/StellaOps.Aoc.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Using Include="Xunit" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
</Project>
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Immutable;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
@@ -9,11 +10,12 @@ using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Attestation.Verification;
|
||||
using StellaOps.Excititor.Export;
|
||||
using StellaOps.Excititor.Export;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Attestation.Dsse;
|
||||
using StellaOps.Excititor.Attestation.Dsse;
|
||||
using StellaOps.Excititor.Attestation.Signing;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Tests;
|
||||
|
||||
@@ -24,23 +26,25 @@ internal static class TestServiceOverrides
|
||||
services.RemoveAll<IVexConnector>();
|
||||
services.RemoveAll<IVexIngestOrchestrator>();
|
||||
services.RemoveAll<IVexConnectorStateRepository>();
|
||||
services.RemoveAll<IVexExportCacheService>();
|
||||
services.RemoveAll<IVexExportDataSource>();
|
||||
services.RemoveAll<IVexExportStore>();
|
||||
services.RemoveAll<IVexCacheIndex>();
|
||||
services.RemoveAll<IVexCacheMaintenance>();
|
||||
services.RemoveAll<IVexAttestationClient>();
|
||||
|
||||
services.AddSingleton<IVexIngestOrchestrator, StubIngestOrchestrator>();
|
||||
services.AddSingleton<IVexConnectorStateRepository, StubConnectorStateRepository>();
|
||||
services.AddSingleton<IVexExportCacheService, StubExportCacheService>();
|
||||
services.RemoveAll<IVexExportCacheService>();
|
||||
services.RemoveAll<IVexExportDataSource>();
|
||||
services.RemoveAll<IVexExportStore>();
|
||||
services.RemoveAll<IVexCacheIndex>();
|
||||
services.RemoveAll<IVexCacheMaintenance>();
|
||||
services.RemoveAll<IVexAttestationClient>();
|
||||
services.RemoveAll<IVexSigner>();
|
||||
|
||||
services.AddSingleton<IVexIngestOrchestrator, StubIngestOrchestrator>();
|
||||
services.AddSingleton<IVexConnectorStateRepository, StubConnectorStateRepository>();
|
||||
services.AddSingleton<IVexExportCacheService, StubExportCacheService>();
|
||||
services.RemoveAll<IExportEngine>();
|
||||
services.AddSingleton<IExportEngine, StubExportEngine>();
|
||||
services.AddSingleton<IVexExportDataSource, StubExportDataSource>();
|
||||
services.AddSingleton<IVexExportStore, StubExportStore>();
|
||||
services.AddSingleton<IVexCacheIndex, StubCacheIndex>();
|
||||
services.AddSingleton<IVexCacheMaintenance, StubCacheMaintenance>();
|
||||
services.AddSingleton<IVexAttestationClient, StubAttestationClient>();
|
||||
services.AddSingleton<IVexExportStore, StubExportStore>();
|
||||
services.AddSingleton<IVexCacheIndex, StubCacheIndex>();
|
||||
services.AddSingleton<IVexCacheMaintenance, StubCacheMaintenance>();
|
||||
services.AddSingleton<IVexAttestationClient, StubAttestationClient>();
|
||||
services.AddSingleton<IVexSigner, StubSigner>();
|
||||
|
||||
services.RemoveAll<IHostedService>();
|
||||
services.AddSingleton<IHostedService, NoopHostedService>();
|
||||
@@ -135,8 +139,8 @@ internal static class TestServiceOverrides
|
||||
=> ValueTask.FromResult(0);
|
||||
}
|
||||
|
||||
private sealed class StubAttestationClient : IVexAttestationClient
|
||||
{
|
||||
private sealed class StubAttestationClient : IVexAttestationClient
|
||||
{
|
||||
public ValueTask<VexAttestationResponse> SignAsync(VexAttestationRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
var envelope = new DsseEnvelope(
|
||||
@@ -168,22 +172,34 @@ internal static class TestServiceOverrides
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubConnectorStateRepository : IVexConnectorStateRepository
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, VexConnectorState> _states = new(StringComparer.Ordinal);
|
||||
|
||||
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
private sealed class StubConnectorStateRepository : IVexConnectorStateRepository
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, VexConnectorState> _states = new(StringComparer.Ordinal);
|
||||
|
||||
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
_states.TryGetValue(connectorId, out var state);
|
||||
return ValueTask.FromResult(state);
|
||||
}
|
||||
|
||||
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
_states[state.ConnectorId] = state;
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
_states[state.ConnectorId] = state;
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
IReadOnlyCollection<VexConnectorState> snapshot = _states.Values.ToList();
|
||||
return ValueTask.FromResult(snapshot);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubSigner : IVexSigner
|
||||
{
|
||||
public ValueTask<VexSignedPayload> SignAsync(ReadOnlyMemory<byte> payload, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(new VexSignedPayload("stub-signature", "stub-key"));
|
||||
}
|
||||
|
||||
private sealed class StubIngestOrchestrator : IVexIngestOrchestrator
|
||||
{
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Aoc;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Tests;
|
||||
|
||||
public sealed class VexGuardSchemaTests
|
||||
{
|
||||
private static readonly AocWriteGuard Guard = new();
|
||||
|
||||
[Fact]
|
||||
public void CycloneDxFixture_CompliesWithGuard()
|
||||
{
|
||||
var result = ValidateCycloneDx();
|
||||
Assert.True(result.IsValid, DescribeViolations(result));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CsafFixture_CompliesWithGuard()
|
||||
{
|
||||
var result = ValidateCsaf();
|
||||
Assert.True(result.IsValid, DescribeViolations(result));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CycloneDxFixture_WithForbiddenField_ProducesErrAoc001()
|
||||
{
|
||||
var result = ValidateCycloneDx(node => node["severity"] = "critical");
|
||||
AssertViolation(result, "ERR_AOC_001", "/severity");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CycloneDxFixture_WithDerivedField_ProducesErrAoc006()
|
||||
{
|
||||
var result = ValidateCycloneDx(node => node["effective_owner"] = "security");
|
||||
AssertViolation(result, "ERR_AOC_006", "/effective_owner");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CycloneDxFixture_WithUnknownField_ProducesErrAoc007()
|
||||
{
|
||||
var result = ValidateCycloneDx(node => node["custom_field"] = 123);
|
||||
AssertViolation(result, "ERR_AOC_007", "/custom_field");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CycloneDxFixture_WithSupersedes_RemainsValid()
|
||||
{
|
||||
var result = ValidateCycloneDx(node => node["supersedes"] = "digest:prev-cdx");
|
||||
Assert.True(result.IsValid, DescribeViolations(result));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CsafFixture_WithSupersedes_RemainsValid()
|
||||
{
|
||||
var result = ValidateCsaf(node => node["supersedes"] = "digest:prev-csaf");
|
||||
Assert.True(result.IsValid, DescribeViolations(result));
|
||||
}
|
||||
|
||||
private static AocGuardResult ValidateCycloneDx(Action<JsonObject>? mutate = null)
|
||||
=> ValidateFixture(CycloneDxRaw, mutate);
|
||||
|
||||
private static AocGuardResult ValidateCsaf(Action<JsonObject>? mutate = null)
|
||||
=> ValidateFixture(CsafRaw, mutate);
|
||||
|
||||
private static AocGuardResult ValidateFixture(string json, Action<JsonObject>? mutate)
|
||||
{
|
||||
var node = JsonNode.Parse(json)!.AsObject();
|
||||
mutate?.Invoke(node);
|
||||
using var document = JsonDocument.Parse(node.ToJsonString());
|
||||
return Guard.Validate(document.RootElement);
|
||||
}
|
||||
|
||||
private static void AssertViolation(AocGuardResult result, string expectedCode, string expectedPath)
|
||||
{
|
||||
Assert.False(result.IsValid);
|
||||
Assert.Contains(result.Violations, violation =>
|
||||
violation.ErrorCode == expectedCode && string.Equals(violation.Path, expectedPath, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
private static string DescribeViolations(AocGuardResult result)
|
||||
=> string.Join(", ", result.Violations.Select(v => $"{v.ErrorCode}:{v.Path}"));
|
||||
|
||||
private const string CycloneDxRaw = """
|
||||
{
|
||||
"tenant": "tests",
|
||||
"source": {
|
||||
"vendor": "cyclonedx",
|
||||
"connector": "cdx",
|
||||
"version": "1.0.0",
|
||||
"stream": "vex-cyclonedx"
|
||||
},
|
||||
"upstream": {
|
||||
"upstream_id": "CDX-2025-0001",
|
||||
"document_version": "2025.11.08",
|
||||
"retrieved_at": "2025-11-08T00:00:00Z",
|
||||
"content_hash": "sha256:cdx",
|
||||
"signature": { "present": false }
|
||||
},
|
||||
"content": {
|
||||
"format": "CycloneDX",
|
||||
"spec_version": "1.6",
|
||||
"raw": {
|
||||
"bomFormat": "CycloneDX",
|
||||
"specVersion": "1.6",
|
||||
"serialNumber": "urn:uuid:12345678-1234-5678-9abc-def012345678",
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"timestamp": "2025-11-08T00:00:00Z",
|
||||
"tools": [
|
||||
{ "vendor": "stellaops", "name": "sample-vex-bot" }
|
||||
]
|
||||
},
|
||||
"vulnerabilities": [
|
||||
{
|
||||
"id": "CVE-2025-0001",
|
||||
"analysis": { "state": "not_affected" },
|
||||
"ratings": [
|
||||
{ "score": 0.0, "method": "cvssv3" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"linkset": {
|
||||
"aliases": [],
|
||||
"references": [],
|
||||
"relationships": [],
|
||||
"products": [],
|
||||
"notes": {},
|
||||
"reconciled_from": []
|
||||
}
|
||||
}
|
||||
""";
|
||||
|
||||
private const string CsafRaw = """
|
||||
{
|
||||
"tenant": "tests",
|
||||
"source": {
|
||||
"vendor": "csaf",
|
||||
"connector": "csaf-json",
|
||||
"version": "1.2.3",
|
||||
"stream": "vex-csaf"
|
||||
},
|
||||
"upstream": {
|
||||
"upstream_id": "CSAF-2025-0002",
|
||||
"document_version": "2025.11.07",
|
||||
"retrieved_at": "2025-11-08T01:10:00Z",
|
||||
"content_hash": "sha256:csaf",
|
||||
"signature": { "present": false }
|
||||
},
|
||||
"content": {
|
||||
"format": "CSAF",
|
||||
"spec_version": "2.0",
|
||||
"raw": {
|
||||
"document": {
|
||||
"category": "csaf_vex",
|
||||
"title": "Sample CSAF VEX",
|
||||
"tracking": {
|
||||
"id": "CSAF-2025-0002",
|
||||
"version": "1",
|
||||
"current_release_date": "2025-11-07T00:00:00Z",
|
||||
"initial_release_date": "2025-11-07T00:00:00Z",
|
||||
"status": "final"
|
||||
}
|
||||
},
|
||||
"product_tree": {
|
||||
"branches": [
|
||||
{
|
||||
"name": "products",
|
||||
"product": {
|
||||
"name": "sample-product",
|
||||
"product_id": "csaf-prod"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"vulnerabilities": [
|
||||
{
|
||||
"cve": "CVE-2025-0002",
|
||||
"product_status": {
|
||||
"fixed": [ "csaf-prod" ]
|
||||
},
|
||||
"threats": [
|
||||
{ "category": "impact", "details": "none" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"linkset": {
|
||||
"aliases": [],
|
||||
"references": [],
|
||||
"relationships": [],
|
||||
"products": [],
|
||||
"notes": {},
|
||||
"reconciled_from": []
|
||||
}
|
||||
}
|
||||
""";
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Net.Http.Json;
|
||||
using System.Text.Json;
|
||||
using EphemeralMongo;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Tests;
|
||||
|
||||
public sealed class VexRawEndpointsTests : IDisposable
|
||||
{
|
||||
private readonly IMongoRunner _runner;
|
||||
private readonly TestWebApplicationFactory _factory;
|
||||
|
||||
public VexRawEndpointsTests()
|
||||
{
|
||||
_runner = MongoRunner.Run(new MongoRunnerOptions { UseSingleNodeReplicaSet = true });
|
||||
|
||||
_factory = new TestWebApplicationFactory(
|
||||
configureConfiguration: configuration =>
|
||||
{
|
||||
configuration.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["Excititor:Storage:Mongo:ConnectionString"] = _runner.ConnectionString,
|
||||
["Excititor:Storage:Mongo:DatabaseName"] = "vex_raw_tests",
|
||||
["Excititor:Storage:Mongo:DefaultTenant"] = "tests",
|
||||
});
|
||||
},
|
||||
configureServices: services =>
|
||||
{
|
||||
TestServiceOverrides.Apply(services);
|
||||
services.AddTestAuthentication();
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IngestListGetAndVerifyFlow()
|
||||
{
|
||||
using var client = _factory.CreateClient(new WebApplicationFactoryClientOptions
|
||||
{
|
||||
AllowAutoRedirect = false
|
||||
});
|
||||
|
||||
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "vex.admin vex.read");
|
||||
client.DefaultRequestHeaders.Add("X-Stella-Tenant", "tests");
|
||||
|
||||
var ingestRequest = BuildVexIngestRequest();
|
||||
var ingestResponse = await client.PostAsJsonAsync("/ingest/vex", ingestRequest);
|
||||
ingestResponse.EnsureSuccessStatusCode();
|
||||
var ingestPayload = await ingestResponse.Content.ReadFromJsonAsync<VexIngestResponse>();
|
||||
Assert.NotNull(ingestPayload);
|
||||
Assert.True(ingestPayload!.Inserted);
|
||||
|
||||
var getResponse = await client.GetAsync($"/vex/raw/{Uri.EscapeDataString(ingestPayload.Digest)}");
|
||||
getResponse.EnsureSuccessStatusCode();
|
||||
var record = await getResponse.Content.ReadFromJsonAsync<VexRawRecordResponse>();
|
||||
Assert.NotNull(record);
|
||||
Assert.Equal(ingestPayload.Digest, record!.Digest);
|
||||
|
||||
var listResponse = await client.GetAsync("/vex/raw?limit=5");
|
||||
listResponse.EnsureSuccessStatusCode();
|
||||
var listPayload = await listResponse.Content.ReadFromJsonAsync<VexRawListResponse>();
|
||||
Assert.NotNull(listPayload);
|
||||
Assert.Contains(listPayload!.Records, summary => summary.Digest == ingestPayload.Digest);
|
||||
|
||||
var verifyRequest = new VexAocVerifyRequest(null, null, 10, null, null);
|
||||
var verifyResponse = await client.PostAsJsonAsync("/aoc/verify", verifyRequest);
|
||||
verifyResponse.EnsureSuccessStatusCode();
|
||||
var verifyPayload = await verifyResponse.Content.ReadFromJsonAsync<VexAocVerifyResponse>();
|
||||
Assert.NotNull(verifyPayload);
|
||||
Assert.True(verifyPayload!.Checked.Vex >= 1);
|
||||
}
|
||||
|
||||
private static VexIngestRequest BuildVexIngestRequest()
|
||||
{
|
||||
using var contentDocument = JsonDocument.Parse("{\"vex\":\"payload\"}");
|
||||
return new VexIngestRequest(
|
||||
ProviderId: "excititor:test",
|
||||
Source: new VexIngestSourceRequest("vendor:test", "connector:test", "1.0.0", "csaf"),
|
||||
Upstream: new VexIngestUpstreamRequest(
|
||||
SourceUri: "https://example.test/vex.json",
|
||||
UpstreamId: "VEX-TEST-001",
|
||||
DocumentVersion: "1",
|
||||
RetrievedAt: DateTimeOffset.UtcNow,
|
||||
ContentHash: "sha256:test",
|
||||
Signature: new VexIngestSignatureRequest(false, null, null, null, null, null),
|
||||
Provenance: new Dictionary<string, string>()),
|
||||
Content: new VexIngestContentRequest("csaf", "2.0", contentDocument.RootElement.Clone(), null),
|
||||
Metadata: new Dictionary<string, string>
|
||||
{
|
||||
["source.vendor"] = "vendor:test"
|
||||
});
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_factory.Dispose();
|
||||
_runner.Dispose();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user