Restructure solution layout by module
This commit is contained in:
master
25
src/Excititor/StellaOps.Excititor.WebService/AGENTS.md
Normal file
25
src/Excititor/StellaOps.Excititor.WebService/AGENTS.md
Normal file
@@ -0,0 +1,25 @@
|
||||
# AGENTS
|
||||
## Role
|
||||
ASP.NET Minimal API surface for Excititor ingest, provider administration, reconciliation, export, and verification flows.
|
||||
## Scope
|
||||
- Program bootstrap, DI wiring for connectors/normalizers/export/attestation/policy/storage.
|
||||
- HTTP endpoints `/excititor/*` with authentication, authorization scopes, request validation, and deterministic responses.
|
||||
- Job orchestration bridges for Worker hand-off (when co-hosted) and offline-friendly configuration.
|
||||
- Observability (structured logs, metrics, tracing) aligned with StellaOps conventions.
|
||||
## Participants
|
||||
- StellaOps.Cli sends `excititor` verbs to this service via token-authenticated HTTPS.
|
||||
- Worker receives scheduled jobs and uses shared infrastructure via common DI extensions.
|
||||
- Authority service provides tokens; WebService enforces scopes before executing operations.
|
||||
## Interfaces & contracts
|
||||
- DTOs for ingest/export requests, run metadata, provider management.
|
||||
- Background job interfaces for ingest/resume/reconcile triggering.
|
||||
- Health/status endpoints exposing pull/export history and current policy revision.
|
||||
## In/Out of scope
|
||||
In: HTTP hosting, request orchestration, DI composition, auth/authorization, logging.
|
||||
Out: long-running ingestion loops (Worker), export rendering (Export module), connector implementations.
|
||||
## Observability & security expectations
|
||||
- Enforce bearer token scopes, enforce audit logging (request/response correlation IDs, provider IDs).
|
||||
- Emit structured events for ingest runs, export invocations, attestation references.
|
||||
- Provide built-in counters/histograms for latency and throughput.
|
||||
## Tests
|
||||
- Minimal API contract/unit tests and integration harness will live in `../StellaOps.Excititor.WebService.Tests`.
|
||||
@@ -0,0 +1,287 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Routing;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Endpoints;
|
||||
|
||||
internal static class IngestEndpoints
|
||||
{
|
||||
private const string AdminScope = "vex.admin";
|
||||
|
||||
public static void MapIngestEndpoints(IEndpointRouteBuilder app)
|
||||
{
|
||||
var group = app.MapGroup("/excititor");
|
||||
|
||||
group.MapPost("/init", HandleInitAsync);
|
||||
group.MapPost("/ingest/run", HandleRunAsync);
|
||||
group.MapPost("/ingest/resume", HandleResumeAsync);
|
||||
group.MapPost("/reconcile", HandleReconcileAsync);
|
||||
}
|
||||
|
||||
internal static async Task<IResult> HandleInitAsync(
|
||||
HttpContext httpContext,
|
||||
ExcititorInitRequest request,
|
||||
IVexIngestOrchestrator orchestrator,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
var providerIds = NormalizeProviders(request.Providers);
|
||||
_ = timeProvider;
|
||||
var options = new IngestInitOptions(providerIds, request.Resume ?? false);
|
||||
|
||||
var summary = await orchestrator.InitializeAsync(options, cancellationToken).ConfigureAwait(false);
|
||||
var message = $"Initialized {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed.";
|
||||
|
||||
return TypedResults.Ok<object>(new
|
||||
{
|
||||
message,
|
||||
runId = summary.RunId,
|
||||
startedAt = summary.StartedAt,
|
||||
completedAt = summary.CompletedAt,
|
||||
providers = summary.Providers.Select(static provider => new
|
||||
{
|
||||
providerId = provider.ProviderId,
|
||||
displayName = provider.DisplayName,
|
||||
status = provider.Status,
|
||||
durationMs = provider.Duration.TotalMilliseconds,
|
||||
error = provider.Error
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
internal static async Task<IResult> HandleRunAsync(
|
||||
HttpContext httpContext,
|
||||
ExcititorIngestRunRequest request,
|
||||
IVexIngestOrchestrator orchestrator,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryParseDateTimeOffset(request.Since, out var since, out var sinceError))
|
||||
{
|
||||
return TypedResults.BadRequest<object>(new { message = sinceError });
|
||||
}
|
||||
|
||||
if (!TryParseTimeSpan(request.Window, out var window, out var windowError))
|
||||
{
|
||||
return TypedResults.BadRequest<object>(new { message = windowError });
|
||||
}
|
||||
|
||||
_ = timeProvider;
|
||||
var providerIds = NormalizeProviders(request.Providers);
|
||||
var options = new IngestRunOptions(
|
||||
providerIds,
|
||||
since,
|
||||
window,
|
||||
request.Force ?? false);
|
||||
|
||||
var summary = await orchestrator.RunAsync(options, cancellationToken).ConfigureAwait(false);
|
||||
var message = $"Ingest run completed for {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed.";
|
||||
|
||||
return TypedResults.Ok<object>(new
|
||||
{
|
||||
message,
|
||||
runId = summary.RunId,
|
||||
startedAt = summary.StartedAt,
|
||||
completedAt = summary.CompletedAt,
|
||||
durationMs = summary.Duration.TotalMilliseconds,
|
||||
providers = summary.Providers.Select(static provider => new
|
||||
{
|
||||
providerId = provider.ProviderId,
|
||||
status = provider.Status,
|
||||
documents = provider.Documents,
|
||||
claims = provider.Claims,
|
||||
startedAt = provider.StartedAt,
|
||||
completedAt = provider.CompletedAt,
|
||||
durationMs = provider.Duration.TotalMilliseconds,
|
||||
lastDigest = provider.LastDigest,
|
||||
lastUpdated = provider.LastUpdated,
|
||||
checkpoint = provider.Checkpoint,
|
||||
error = provider.Error
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
internal static async Task<IResult> HandleResumeAsync(
|
||||
HttpContext httpContext,
|
||||
ExcititorIngestResumeRequest request,
|
||||
IVexIngestOrchestrator orchestrator,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
_ = timeProvider;
|
||||
var providerIds = NormalizeProviders(request.Providers);
|
||||
var options = new IngestResumeOptions(providerIds, request.Checkpoint);
|
||||
|
||||
var summary = await orchestrator.ResumeAsync(options, cancellationToken).ConfigureAwait(false);
|
||||
var message = $"Resume run completed for {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed.";
|
||||
|
||||
return TypedResults.Ok<object>(new
|
||||
{
|
||||
message,
|
||||
runId = summary.RunId,
|
||||
startedAt = summary.StartedAt,
|
||||
completedAt = summary.CompletedAt,
|
||||
durationMs = summary.Duration.TotalMilliseconds,
|
||||
providers = summary.Providers.Select(static provider => new
|
||||
{
|
||||
providerId = provider.ProviderId,
|
||||
status = provider.Status,
|
||||
documents = provider.Documents,
|
||||
claims = provider.Claims,
|
||||
startedAt = provider.StartedAt,
|
||||
completedAt = provider.CompletedAt,
|
||||
durationMs = provider.Duration.TotalMilliseconds,
|
||||
since = provider.Since,
|
||||
checkpoint = provider.Checkpoint,
|
||||
error = provider.Error
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
internal static async Task<IResult> HandleReconcileAsync(
|
||||
HttpContext httpContext,
|
||||
ExcititorReconcileRequest request,
|
||||
IVexIngestOrchestrator orchestrator,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryParseTimeSpan(request.MaxAge, out var maxAge, out var error))
|
||||
{
|
||||
return TypedResults.BadRequest<object>(new { message = error });
|
||||
}
|
||||
|
||||
_ = timeProvider;
|
||||
var providerIds = NormalizeProviders(request.Providers);
|
||||
var options = new ReconcileOptions(providerIds, maxAge);
|
||||
|
||||
var summary = await orchestrator.ReconcileAsync(options, cancellationToken).ConfigureAwait(false);
|
||||
var message = $"Reconcile completed for {summary.ProviderCount} provider(s); {summary.ReconciledCount} reconciled, {summary.SkippedCount} skipped, {summary.FailureCount} failed.";
|
||||
|
||||
return TypedResults.Ok<object>(new
|
||||
{
|
||||
message,
|
||||
runId = summary.RunId,
|
||||
startedAt = summary.StartedAt,
|
||||
completedAt = summary.CompletedAt,
|
||||
durationMs = summary.Duration.TotalMilliseconds,
|
||||
providers = summary.Providers.Select(static provider => new
|
||||
{
|
||||
providerId = provider.ProviderId,
|
||||
status = provider.Status,
|
||||
action = provider.Action,
|
||||
lastUpdated = provider.LastUpdated,
|
||||
threshold = provider.Threshold,
|
||||
documents = provider.Documents,
|
||||
claims = provider.Claims,
|
||||
error = provider.Error
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
internal static ImmutableArray<string> NormalizeProviders(IReadOnlyCollection<string>? providers)
|
||||
{
|
||||
if (providers is null || providers.Count == 0)
|
||||
{
|
||||
return ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
var set = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var provider in providers)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(provider))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
set.Add(provider.Trim());
|
||||
}
|
||||
|
||||
return set.ToImmutableArray();
|
||||
}
|
||||
|
||||
internal static bool TryParseDateTimeOffset(string? value, out DateTimeOffset? result, out string? error)
|
||||
{
|
||||
result = null;
|
||||
error = null;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (DateTimeOffset.TryParse(
|
||||
value.Trim(),
|
||||
CultureInfo.InvariantCulture,
|
||||
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
|
||||
out var parsed))
|
||||
{
|
||||
result = parsed;
|
||||
return true;
|
||||
}
|
||||
|
||||
error = "Invalid 'since' value. Use ISO-8601 format (e.g. 2025-10-19T12:30:00Z).";
|
||||
return false;
|
||||
}
|
||||
|
||||
internal static bool TryParseTimeSpan(string? value, out TimeSpan? result, out string? error)
|
||||
{
|
||||
result = null;
|
||||
error = null;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TimeSpan.TryParse(value.Trim(), CultureInfo.InvariantCulture, out var parsed) && parsed >= TimeSpan.Zero)
|
||||
{
|
||||
result = parsed;
|
||||
return true;
|
||||
}
|
||||
|
||||
error = "Invalid duration value. Use TimeSpan format (e.g. 1.00:00:00).";
|
||||
return false;
|
||||
}
|
||||
|
||||
internal sealed record ExcititorInitRequest(IReadOnlyList<string>? Providers, bool? Resume);
|
||||
|
||||
internal sealed record ExcititorIngestRunRequest(
|
||||
IReadOnlyList<string>? Providers,
|
||||
string? Since,
|
||||
string? Window,
|
||||
bool? Force);
|
||||
|
||||
internal sealed record ExcititorIngestResumeRequest(
|
||||
IReadOnlyList<string>? Providers,
|
||||
string? Checkpoint);
|
||||
|
||||
internal sealed record ExcititorReconcileRequest(
|
||||
IReadOnlyList<string>? Providers,
|
||||
string? MaxAge);
|
||||
}
|
||||
@@ -0,0 +1,388 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Export;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Endpoints;
|
||||
|
||||
internal static class MirrorEndpoints
|
||||
{
|
||||
public static void MapMirrorEndpoints(WebApplication app)
|
||||
{
|
||||
var group = app.MapGroup("/excititor/mirror");
|
||||
|
||||
group.MapGet("/domains", HandleListDomainsAsync);
|
||||
group.MapGet("/domains/{domainId}", HandleDomainDetailAsync);
|
||||
group.MapGet("/domains/{domainId}/index", HandleDomainIndexAsync);
|
||||
group.MapGet("/domains/{domainId}/exports/{exportKey}", HandleExportMetadataAsync);
|
||||
group.MapGet("/domains/{domainId}/exports/{exportKey}/download", HandleExportDownloadAsync);
|
||||
}
|
||||
|
||||
private static async Task<IResult> HandleListDomainsAsync(
|
||||
HttpContext httpContext,
|
||||
IOptions<MirrorDistributionOptions> options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var domains = options.Value.Domains
|
||||
.Select(static domain => new MirrorDomainSummary(
|
||||
domain.Id,
|
||||
string.IsNullOrWhiteSpace(domain.DisplayName) ? domain.Id : domain.DisplayName,
|
||||
domain.RequireAuthentication,
|
||||
Math.Max(domain.MaxIndexRequestsPerHour, 0),
|
||||
Math.Max(domain.MaxDownloadRequestsPerHour, 0)))
|
||||
.ToArray();
|
||||
|
||||
await WriteJsonAsync(httpContext, new MirrorDomainListResponse(domains), StatusCodes.Status200OK, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
private static async Task<IResult> HandleDomainDetailAsync(
|
||||
string domainId,
|
||||
HttpContext httpContext,
|
||||
IOptions<MirrorDistributionOptions> options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryFindDomain(options.Value, domainId, out var domain))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
var response = new MirrorDomainDetail(
|
||||
domain.Id,
|
||||
string.IsNullOrWhiteSpace(domain.DisplayName) ? domain.Id : domain.DisplayName,
|
||||
domain.RequireAuthentication,
|
||||
Math.Max(domain.MaxIndexRequestsPerHour, 0),
|
||||
Math.Max(domain.MaxDownloadRequestsPerHour, 0),
|
||||
domain.Exports.Select(static export => export.Key).OrderBy(static key => key, StringComparer.Ordinal).ToImmutableArray());
|
||||
|
||||
await WriteJsonAsync(httpContext, response, StatusCodes.Status200OK, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
private static async Task<IResult> HandleDomainIndexAsync(
|
||||
string domainId,
|
||||
HttpContext httpContext,
|
||||
IOptions<MirrorDistributionOptions> options,
|
||||
MirrorRateLimiter rateLimiter,
|
||||
IVexExportStore exportStore,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryFindDomain(options.Value, domainId, out var domain))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
if (domain.RequireAuthentication && (httpContext.User?.Identity?.IsAuthenticated is not true))
|
||||
{
|
||||
return Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!rateLimiter.TryAcquire(domain.Id, "index", Math.Max(domain.MaxIndexRequestsPerHour, 0), out var retryAfter))
|
||||
{
|
||||
if (retryAfter is { } retry)
|
||||
{
|
||||
httpContext.Response.Headers.RetryAfter = ((int)Math.Ceiling(retry.TotalSeconds)).ToString(CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
await WritePlainTextAsync(httpContext, "mirror index quota exceeded", StatusCodes.Status429TooManyRequests, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
var resolvedExports = new List<MirrorExportIndexEntry>();
|
||||
foreach (var exportOption in domain.Exports)
|
||||
{
|
||||
if (!MirrorExportPlanner.TryBuild(exportOption, out var plan, out var error))
|
||||
{
|
||||
resolvedExports.Add(new MirrorExportIndexEntry(
|
||||
exportOption.Key,
|
||||
null,
|
||||
null,
|
||||
exportOption.Format,
|
||||
null,
|
||||
null,
|
||||
0,
|
||||
null,
|
||||
null,
|
||||
error ?? "invalid_export_configuration"));
|
||||
continue;
|
||||
}
|
||||
|
||||
var manifest = await exportStore.FindAsync(plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (manifest is null)
|
||||
{
|
||||
resolvedExports.Add(new MirrorExportIndexEntry(
|
||||
exportOption.Key,
|
||||
null,
|
||||
plan.Signature.Value,
|
||||
plan.Format.ToString().ToLowerInvariant(),
|
||||
null,
|
||||
null,
|
||||
0,
|
||||
null,
|
||||
null,
|
||||
"manifest_not_found"));
|
||||
continue;
|
||||
}
|
||||
|
||||
resolvedExports.Add(new MirrorExportIndexEntry(
|
||||
exportOption.Key,
|
||||
manifest.ExportId,
|
||||
manifest.QuerySignature.Value,
|
||||
manifest.Format.ToString().ToLowerInvariant(),
|
||||
manifest.CreatedAt,
|
||||
manifest.Artifact,
|
||||
manifest.SizeBytes,
|
||||
manifest.ConsensusRevision,
|
||||
manifest.Attestation is null ? null : new MirrorExportAttestation(manifest.Attestation.PredicateType, manifest.Attestation.Rekor?.Location, manifest.Attestation.EnvelopeDigest, manifest.Attestation.SignedAt),
|
||||
null));
|
||||
}
|
||||
|
||||
var indexResponse = new MirrorDomainIndex(
|
||||
domain.Id,
|
||||
string.IsNullOrWhiteSpace(domain.DisplayName) ? domain.Id : domain.DisplayName,
|
||||
timeProvider.GetUtcNow(),
|
||||
resolvedExports.ToImmutableArray());
|
||||
|
||||
await WriteJsonAsync(httpContext, indexResponse, StatusCodes.Status200OK, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
private static async Task<IResult> HandleExportMetadataAsync(
|
||||
string domainId,
|
||||
string exportKey,
|
||||
HttpContext httpContext,
|
||||
IOptions<MirrorDistributionOptions> options,
|
||||
MirrorRateLimiter rateLimiter,
|
||||
IVexExportStore exportStore,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryFindDomain(options.Value, domainId, out var domain))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
if (domain.RequireAuthentication && (httpContext.User?.Identity?.IsAuthenticated is not true))
|
||||
{
|
||||
return Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!TryFindExport(domain, exportKey, out var exportOptions))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
if (!MirrorExportPlanner.TryBuild(exportOptions, out var plan, out var error))
|
||||
{
|
||||
await WritePlainTextAsync(httpContext, error ?? "invalid_export_configuration", StatusCodes.Status503ServiceUnavailable, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
var manifest = await exportStore.FindAsync(plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false);
|
||||
if (manifest is null)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
var payload = new MirrorExportMetadata(
|
||||
domain.Id,
|
||||
exportOptions.Key,
|
||||
manifest.ExportId,
|
||||
manifest.QuerySignature.Value,
|
||||
manifest.Format.ToString().ToLowerInvariant(),
|
||||
manifest.CreatedAt,
|
||||
manifest.Artifact,
|
||||
manifest.SizeBytes,
|
||||
manifest.SourceProviders,
|
||||
manifest.Attestation is null ? null : new MirrorExportAttestation(manifest.Attestation.PredicateType, manifest.Attestation.Rekor?.Location, manifest.Attestation.EnvelopeDigest, manifest.Attestation.SignedAt));
|
||||
|
||||
await WriteJsonAsync(httpContext, payload, StatusCodes.Status200OK, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
private static async Task<IResult> HandleExportDownloadAsync(
|
||||
string domainId,
|
||||
string exportKey,
|
||||
HttpContext httpContext,
|
||||
IOptions<MirrorDistributionOptions> options,
|
||||
MirrorRateLimiter rateLimiter,
|
||||
IVexExportStore exportStore,
|
||||
IEnumerable<IVexArtifactStore> artifactStores,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryFindDomain(options.Value, domainId, out var domain))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
if (domain.RequireAuthentication && (httpContext.User?.Identity?.IsAuthenticated is not true))
|
||||
{
|
||||
return Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!rateLimiter.TryAcquire(domain.Id, "download", Math.Max(domain.MaxDownloadRequestsPerHour, 0), out var retryAfter))
|
||||
{
|
||||
if (retryAfter is { } retry)
|
||||
{
|
||||
httpContext.Response.Headers.RetryAfter = ((int)Math.Ceiling(retry.TotalSeconds)).ToString(CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
await WritePlainTextAsync(httpContext, "mirror download quota exceeded", StatusCodes.Status429TooManyRequests, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
if (!TryFindExport(domain, exportKey, out var exportOptions) || !MirrorExportPlanner.TryBuild(exportOptions, out var plan, out _))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
var manifest = await exportStore.FindAsync(plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false);
|
||||
if (manifest is null)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
Stream? contentStream = null;
|
||||
foreach (var store in artifactStores)
|
||||
{
|
||||
contentStream = await store.OpenReadAsync(manifest.Artifact, cancellationToken).ConfigureAwait(false);
|
||||
if (contentStream is not null)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (contentStream is null)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
await using (contentStream.ConfigureAwait(false))
|
||||
{
|
||||
var contentType = ResolveContentType(manifest.Format);
|
||||
httpContext.Response.StatusCode = StatusCodes.Status200OK;
|
||||
httpContext.Response.ContentType = contentType;
|
||||
httpContext.Response.Headers.ContentDisposition = FormattableString.Invariant($"attachment; filename=\"{BuildDownloadFileName(domain.Id, exportOptions.Key, manifest.Format)}\"");
|
||||
|
||||
await contentStream.CopyToAsync(httpContext.Response.Body, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
private static bool TryFindDomain(MirrorDistributionOptions options, string domainId, out MirrorDomainOptions domain)
|
||||
{
|
||||
domain = options.Domains.FirstOrDefault(d => string.Equals(d.Id, domainId, StringComparison.OrdinalIgnoreCase))!;
|
||||
return domain is not null;
|
||||
}
|
||||
|
||||
private static bool TryFindExport(MirrorDomainOptions domain, string exportKey, out MirrorExportOptions export)
|
||||
{
|
||||
export = domain.Exports.FirstOrDefault(e => string.Equals(e.Key, exportKey, StringComparison.OrdinalIgnoreCase))!;
|
||||
return export is not null;
|
||||
}
|
||||
|
||||
private static string ResolveContentType(VexExportFormat format)
|
||||
=> format switch
|
||||
{
|
||||
VexExportFormat.Json => "application/json",
|
||||
VexExportFormat.JsonLines => "application/jsonl",
|
||||
VexExportFormat.OpenVex => "application/json",
|
||||
VexExportFormat.Csaf => "application/json",
|
||||
_ => "application/octet-stream",
|
||||
};
|
||||
|
||||
private static string BuildDownloadFileName(string domainId, string exportKey, VexExportFormat format)
|
||||
{
|
||||
var builder = new StringBuilder(domainId.Length + exportKey.Length + 8);
|
||||
builder.Append(domainId).Append('-').Append(exportKey);
|
||||
builder.Append(format switch
|
||||
{
|
||||
VexExportFormat.Json => ".json",
|
||||
VexExportFormat.JsonLines => ".jsonl",
|
||||
VexExportFormat.OpenVex => ".openvex.json",
|
||||
VexExportFormat.Csaf => ".csaf.json",
|
||||
_ => ".bin",
|
||||
});
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
private static async Task WritePlainTextAsync(HttpContext context, string message, int statusCode, CancellationToken cancellationToken)
|
||||
{
|
||||
context.Response.StatusCode = statusCode;
|
||||
context.Response.ContentType = "text/plain";
|
||||
await context.Response.WriteAsync(message, cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task WriteJsonAsync<T>(HttpContext context, T payload, int statusCode, CancellationToken cancellationToken)
|
||||
{
|
||||
context.Response.StatusCode = statusCode;
|
||||
context.Response.ContentType = "application/json";
|
||||
var json = VexCanonicalJsonSerializer.Serialize(payload);
|
||||
await context.Response.WriteAsync(json, cancellationToken);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
internal sealed record MirrorDomainListResponse(IReadOnlyList<MirrorDomainSummary> Domains);
|
||||
|
||||
internal sealed record MirrorDomainSummary(
|
||||
string Id,
|
||||
string DisplayName,
|
||||
bool RequireAuthentication,
|
||||
int MaxIndexRequestsPerHour,
|
||||
int MaxDownloadRequestsPerHour);
|
||||
|
||||
internal sealed record MirrorDomainDetail(
|
||||
string Id,
|
||||
string DisplayName,
|
||||
bool RequireAuthentication,
|
||||
int MaxIndexRequestsPerHour,
|
||||
int MaxDownloadRequestsPerHour,
|
||||
IReadOnlyList<string> Exports);
|
||||
|
||||
internal sealed record MirrorDomainIndex(
|
||||
string Id,
|
||||
string DisplayName,
|
||||
DateTimeOffset GeneratedAt,
|
||||
IReadOnlyList<MirrorExportIndexEntry> Exports);
|
||||
|
||||
internal sealed record MirrorExportIndexEntry(
|
||||
string ExportKey,
|
||||
string? ExportId,
|
||||
string? QuerySignature,
|
||||
string Format,
|
||||
DateTimeOffset? CreatedAt,
|
||||
VexContentAddress? Artifact,
|
||||
long SizeBytes,
|
||||
string? ConsensusRevision,
|
||||
MirrorExportAttestation? Attestation,
|
||||
string? Status);
|
||||
|
||||
internal sealed record MirrorExportAttestation(
|
||||
string PredicateType,
|
||||
string? RekorLocation,
|
||||
string? EnvelopeDigest,
|
||||
DateTimeOffset? SignedAt);
|
||||
|
||||
internal sealed record MirrorExportMetadata(
|
||||
string DomainId,
|
||||
string ExportKey,
|
||||
string ExportId,
|
||||
string QuerySignature,
|
||||
string Format,
|
||||
DateTimeOffset CreatedAt,
|
||||
VexContentAddress Artifact,
|
||||
long SizeBytes,
|
||||
IReadOnlyList<string> SourceProviders,
|
||||
MirrorExportAttestation? Attestation);
|
||||
@@ -0,0 +1,512 @@
|
||||
namespace StellaOps.Excititor.WebService.Endpoints;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Excititor.Attestation;
|
||||
using StellaOps.Excititor.Attestation.Dsse;
|
||||
using StellaOps.Excititor.Attestation.Signing;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Policy;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
|
||||
internal static class ResolveEndpoint
|
||||
{
|
||||
private const int MaxSubjectPairs = 256;
|
||||
private const string ReadScope = "vex.read";
|
||||
|
||||
public static void MapResolveEndpoint(WebApplication app)
|
||||
{
|
||||
app.MapPost("/excititor/resolve", HandleResolveAsync);
|
||||
}
|
||||
|
||||
private static async Task<IResult> HandleResolveAsync(
|
||||
VexResolveRequest request,
|
||||
HttpContext httpContext,
|
||||
IVexClaimStore claimStore,
|
||||
IVexConsensusStore consensusStore,
|
||||
IVexProviderStore providerStore,
|
||||
IVexPolicyProvider policyProvider,
|
||||
TimeProvider timeProvider,
|
||||
ILoggerFactory loggerFactory,
|
||||
IVexAttestationClient? attestationClient,
|
||||
IVexSigner? signer,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(httpContext, ReadScope);
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (request is null)
|
||||
{
|
||||
return Results.BadRequest("Request payload is required.");
|
||||
}
|
||||
|
||||
var logger = loggerFactory.CreateLogger("ResolveEndpoint");
|
||||
|
||||
var productKeys = NormalizeValues(request.ProductKeys, request.Purls);
|
||||
var vulnerabilityIds = NormalizeValues(request.VulnerabilityIds);
|
||||
|
||||
if (productKeys.Count == 0)
|
||||
{
|
||||
await WritePlainTextAsync(httpContext, "At least one productKey or purl must be provided.", StatusCodes.Status400BadRequest, cancellationToken);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
if (vulnerabilityIds.Count == 0)
|
||||
{
|
||||
await WritePlainTextAsync(httpContext, "At least one vulnerabilityId must be provided.", StatusCodes.Status400BadRequest, cancellationToken);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
var pairCount = (long)productKeys.Count * vulnerabilityIds.Count;
|
||||
if (pairCount > MaxSubjectPairs)
|
||||
{
|
||||
await WritePlainTextAsync(httpContext, FormattableString.Invariant($"A maximum of {MaxSubjectPairs} subject pairs are allowed per request."), StatusCodes.Status400BadRequest, cancellationToken);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
var snapshot = policyProvider.GetSnapshot();
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(request.PolicyRevisionId) &&
|
||||
!string.Equals(request.PolicyRevisionId.Trim(), snapshot.RevisionId, StringComparison.Ordinal))
|
||||
{
|
||||
var conflictPayload = new
|
||||
{
|
||||
message = $"Requested policy revision '{request.PolicyRevisionId}' does not match active revision '{snapshot.RevisionId}'.",
|
||||
activeRevision = snapshot.RevisionId,
|
||||
requestedRevision = request.PolicyRevisionId,
|
||||
};
|
||||
await WriteJsonAsync(httpContext, conflictPayload, StatusCodes.Status409Conflict, cancellationToken);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
var resolver = new VexConsensusResolver(snapshot.ConsensusPolicy);
|
||||
var resolvedAt = timeProvider.GetUtcNow();
|
||||
var providerCache = new Dictionary<string, VexProvider>(StringComparer.Ordinal);
|
||||
var results = new List<VexResolveResult>((int)pairCount);
|
||||
|
||||
foreach (var productKey in productKeys)
|
||||
{
|
||||
foreach (var vulnerabilityId in vulnerabilityIds)
|
||||
{
|
||||
var claims = await claimStore.FindAsync(vulnerabilityId, productKey, since: null, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var claimArray = claims.Count == 0 ? Array.Empty<VexClaim>() : claims.ToArray();
|
||||
var signals = AggregateSignals(claimArray);
|
||||
var providers = await LoadProvidersAsync(claimArray, providerStore, providerCache, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
var product = ResolveProduct(claimArray, productKey);
|
||||
var calculatedAt = timeProvider.GetUtcNow();
|
||||
|
||||
var resolution = resolver.Resolve(new VexConsensusRequest(
|
||||
vulnerabilityId,
|
||||
product,
|
||||
claimArray,
|
||||
providers,
|
||||
calculatedAt,
|
||||
snapshot.ConsensusOptions.WeightCeiling,
|
||||
signals,
|
||||
snapshot.RevisionId,
|
||||
snapshot.Digest));
|
||||
|
||||
var consensus = resolution.Consensus;
|
||||
|
||||
if (!string.Equals(consensus.PolicyVersion, snapshot.Version, StringComparison.Ordinal) ||
|
||||
!string.Equals(consensus.PolicyRevisionId, snapshot.RevisionId, StringComparison.Ordinal) ||
|
||||
!string.Equals(consensus.PolicyDigest, snapshot.Digest, StringComparison.Ordinal))
|
||||
{
|
||||
consensus = new VexConsensus(
|
||||
consensus.VulnerabilityId,
|
||||
consensus.Product,
|
||||
consensus.Status,
|
||||
consensus.CalculatedAt,
|
||||
consensus.Sources,
|
||||
consensus.Conflicts,
|
||||
consensus.Signals,
|
||||
snapshot.Version,
|
||||
consensus.Summary,
|
||||
snapshot.RevisionId,
|
||||
snapshot.Digest);
|
||||
}
|
||||
|
||||
await consensusStore.SaveAsync(consensus, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var payload = PreparePayload(consensus);
|
||||
var contentSignature = await TrySignAsync(signer, payload, logger, cancellationToken).ConfigureAwait(false);
|
||||
var attestation = await BuildAttestationAsync(
|
||||
attestationClient,
|
||||
consensus,
|
||||
snapshot,
|
||||
payload,
|
||||
logger,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var decisions = resolution.DecisionLog.IsDefault
|
||||
? Array.Empty<VexConsensusDecisionTelemetry>()
|
||||
: resolution.DecisionLog.ToArray();
|
||||
|
||||
results.Add(new VexResolveResult(
|
||||
consensus.VulnerabilityId,
|
||||
consensus.Product.Key,
|
||||
consensus.Status,
|
||||
consensus.CalculatedAt,
|
||||
consensus.Sources,
|
||||
consensus.Conflicts,
|
||||
consensus.Signals,
|
||||
consensus.Summary,
|
||||
consensus.PolicyRevisionId ?? snapshot.RevisionId,
|
||||
consensus.PolicyVersion ?? snapshot.Version,
|
||||
consensus.PolicyDigest ?? snapshot.Digest,
|
||||
decisions,
|
||||
new VexResolveEnvelope(
|
||||
payload.Artifact,
|
||||
contentSignature,
|
||||
attestation.Metadata,
|
||||
attestation.Envelope,
|
||||
attestation.Signature)));
|
||||
}
|
||||
}
|
||||
|
||||
var policy = new VexResolvePolicy(
|
||||
snapshot.RevisionId,
|
||||
snapshot.Version,
|
||||
snapshot.Digest,
|
||||
request.PolicyRevisionId?.Trim());
|
||||
|
||||
var response = new VexResolveResponse(resolvedAt, policy, results);
|
||||
await WriteJsonAsync(httpContext, response, StatusCodes.Status200OK, cancellationToken);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
private static List<string> NormalizeValues(params IReadOnlyList<string>?[] sources)
|
||||
{
|
||||
var result = new List<string>();
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var source in sources)
|
||||
{
|
||||
if (source is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var value in source)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var normalized = value.Trim();
|
||||
if (seen.Add(normalized))
|
||||
{
|
||||
result.Add(normalized);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static VexSignalSnapshot? AggregateSignals(IReadOnlyList<VexClaim> claims)
|
||||
{
|
||||
if (claims.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
VexSeveritySignal? bestSeverity = null;
|
||||
double? bestScore = null;
|
||||
bool kevPresent = false;
|
||||
bool kevTrue = false;
|
||||
double? bestEpss = null;
|
||||
|
||||
foreach (var claim in claims)
|
||||
{
|
||||
if (claim.Signals is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var severity = claim.Signals.Severity;
|
||||
if (severity is not null)
|
||||
{
|
||||
var score = severity.Score;
|
||||
if (bestSeverity is null ||
|
||||
(score is not null && (bestScore is null || score.Value > bestScore.Value)) ||
|
||||
(score is null && bestScore is null && !string.IsNullOrWhiteSpace(severity.Label) && string.IsNullOrWhiteSpace(bestSeverity.Label)))
|
||||
{
|
||||
bestSeverity = severity;
|
||||
bestScore = severity.Score;
|
||||
}
|
||||
}
|
||||
|
||||
if (claim.Signals.Kev is { } kevValue)
|
||||
{
|
||||
kevPresent = true;
|
||||
if (kevValue)
|
||||
{
|
||||
kevTrue = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (claim.Signals.Epss is { } epss)
|
||||
{
|
||||
if (bestEpss is null || epss > bestEpss.Value)
|
||||
{
|
||||
bestEpss = epss;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (bestSeverity is null && !kevPresent && bestEpss is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
bool? kev = kevTrue ? true : (kevPresent ? false : null);
|
||||
return new VexSignalSnapshot(bestSeverity, kev, bestEpss);
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyDictionary<string, VexProvider>> LoadProvidersAsync(
|
||||
IReadOnlyList<VexClaim> claims,
|
||||
IVexProviderStore providerStore,
|
||||
IDictionary<string, VexProvider> cache,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (claims.Count == 0)
|
||||
{
|
||||
return ImmutableDictionary<string, VexProvider>.Empty;
|
||||
}
|
||||
|
||||
var builder = ImmutableDictionary.CreateBuilder<string, VexProvider>(StringComparer.Ordinal);
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var providerId in claims.Select(claim => claim.ProviderId))
|
||||
{
|
||||
if (!seen.Add(providerId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (cache.TryGetValue(providerId, out var cached))
|
||||
{
|
||||
builder[providerId] = cached;
|
||||
continue;
|
||||
}
|
||||
|
||||
var provider = await providerStore.FindAsync(providerId, cancellationToken).ConfigureAwait(false);
|
||||
if (provider is not null)
|
||||
{
|
||||
cache[providerId] = provider;
|
||||
builder[providerId] = provider;
|
||||
}
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
private static VexProduct ResolveProduct(IReadOnlyList<VexClaim> claims, string productKey)
|
||||
{
|
||||
if (claims.Count > 0)
|
||||
{
|
||||
return claims[0].Product;
|
||||
}
|
||||
|
||||
var inferredPurl = productKey.StartsWith("pkg:", StringComparison.OrdinalIgnoreCase) ? productKey : null;
|
||||
return new VexProduct(productKey, name: null, version: null, purl: inferredPurl);
|
||||
}
|
||||
|
||||
private static ConsensusPayload PreparePayload(VexConsensus consensus)
|
||||
{
|
||||
var canonicalJson = VexCanonicalJsonSerializer.Serialize(consensus);
|
||||
var bytes = Encoding.UTF8.GetBytes(canonicalJson);
|
||||
var digest = SHA256.HashData(bytes);
|
||||
var digestHex = Convert.ToHexString(digest).ToLowerInvariant();
|
||||
var address = new VexContentAddress("sha256", digestHex);
|
||||
return new ConsensusPayload(address, bytes, canonicalJson);
|
||||
}
|
||||
|
||||
private static async ValueTask<ResolveSignature?> TrySignAsync(
|
||||
IVexSigner? signer,
|
||||
ConsensusPayload payload,
|
||||
ILogger logger,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (signer is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var signature = await signer.SignAsync(payload.Bytes, cancellationToken).ConfigureAwait(false);
|
||||
return new ResolveSignature(signature.Signature, signature.KeyId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogWarning(ex, "Failed to sign resolve payload {Digest}", payload.Artifact.ToUri());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static async ValueTask<ResolveAttestation> BuildAttestationAsync(
|
||||
IVexAttestationClient? attestationClient,
|
||||
VexConsensus consensus,
|
||||
VexPolicySnapshot snapshot,
|
||||
ConsensusPayload payload,
|
||||
ILogger logger,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (attestationClient is null)
|
||||
{
|
||||
return new ResolveAttestation(null, null, null);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var exportId = BuildAttestationExportId(consensus.VulnerabilityId, consensus.Product.Key);
|
||||
var filters = new[]
|
||||
{
|
||||
new KeyValuePair<string, string>("vulnerabilityId", consensus.VulnerabilityId),
|
||||
new KeyValuePair<string, string>("productKey", consensus.Product.Key),
|
||||
new KeyValuePair<string, string>("policyRevisionId", snapshot.RevisionId),
|
||||
};
|
||||
|
||||
var querySignature = VexQuerySignature.FromFilters(filters);
|
||||
var providerIds = consensus.Sources
|
||||
.Select(source => source.ProviderId)
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
var metadataBuilder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
|
||||
metadataBuilder["consensusDigest"] = payload.Artifact.ToUri();
|
||||
metadataBuilder["policyRevisionId"] = snapshot.RevisionId;
|
||||
metadataBuilder["policyVersion"] = snapshot.Version;
|
||||
if (!string.IsNullOrWhiteSpace(snapshot.Digest))
|
||||
{
|
||||
metadataBuilder["policyDigest"] = snapshot.Digest;
|
||||
}
|
||||
|
||||
var response = await attestationClient.SignAsync(new VexAttestationRequest(
|
||||
exportId,
|
||||
querySignature,
|
||||
payload.Artifact,
|
||||
VexExportFormat.Json,
|
||||
consensus.CalculatedAt,
|
||||
providerIds,
|
||||
metadataBuilder.ToImmutable()), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var envelopeJson = response.Diagnostics.TryGetValue("envelope", out var envelopeValue)
|
||||
? envelopeValue
|
||||
: null;
|
||||
|
||||
ResolveSignature? signature = null;
|
||||
if (!string.IsNullOrWhiteSpace(envelopeJson))
|
||||
{
|
||||
try
|
||||
{
|
||||
var envelope = JsonSerializer.Deserialize<DsseEnvelope>(envelopeJson);
|
||||
var dsseSignature = envelope?.Signatures?.FirstOrDefault();
|
||||
if (dsseSignature is not null)
|
||||
{
|
||||
signature = new ResolveSignature(dsseSignature.Signature, dsseSignature.KeyId);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogDebug(ex, "Failed to deserialize DSSE envelope for resolve export {ExportId}", exportId);
|
||||
}
|
||||
}
|
||||
|
||||
return new ResolveAttestation(response.Attestation, envelopeJson, signature);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogWarning(ex, "Unable to produce attestation for {VulnerabilityId}/{ProductKey}", consensus.VulnerabilityId, consensus.Product.Key);
|
||||
return new ResolveAttestation(null, null, null);
|
||||
}
|
||||
}
|
||||
|
||||
private static string BuildAttestationExportId(string vulnerabilityId, string productKey)
|
||||
{
|
||||
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(productKey));
|
||||
var digest = Convert.ToHexString(hash).ToLowerInvariant();
|
||||
return FormattableString.Invariant($"resolve/{vulnerabilityId}/{digest}");
|
||||
}
|
||||
|
||||
private sealed record ConsensusPayload(VexContentAddress Artifact, byte[] Bytes, string CanonicalJson);
|
||||
|
||||
private static async Task WritePlainTextAsync(HttpContext context, string message, int statusCode, CancellationToken cancellationToken)
|
||||
{
|
||||
context.Response.StatusCode = statusCode;
|
||||
context.Response.ContentType = "text/plain";
|
||||
await context.Response.WriteAsync(message, cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task WriteJsonAsync<T>(HttpContext context, T payload, int statusCode, CancellationToken cancellationToken)
|
||||
{
|
||||
context.Response.StatusCode = statusCode;
|
||||
context.Response.ContentType = "application/json";
|
||||
var json = VexCanonicalJsonSerializer.Serialize(payload);
|
||||
await context.Response.WriteAsync(json, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record VexResolveRequest(
|
||||
IReadOnlyList<string>? ProductKeys,
|
||||
IReadOnlyList<string>? Purls,
|
||||
IReadOnlyList<string>? VulnerabilityIds,
|
||||
string? PolicyRevisionId);
|
||||
|
||||
internal sealed record VexResolvePolicy(
|
||||
string ActiveRevisionId,
|
||||
string Version,
|
||||
string Digest,
|
||||
string? RequestedRevisionId);
|
||||
|
||||
internal sealed record VexResolveResponse(
|
||||
DateTimeOffset ResolvedAt,
|
||||
VexResolvePolicy Policy,
|
||||
IReadOnlyList<VexResolveResult> Results);
|
||||
|
||||
internal sealed record VexResolveResult(
|
||||
string VulnerabilityId,
|
||||
string ProductKey,
|
||||
VexConsensusStatus Status,
|
||||
DateTimeOffset CalculatedAt,
|
||||
IReadOnlyList<VexConsensusSource> Sources,
|
||||
IReadOnlyList<VexConsensusConflict> Conflicts,
|
||||
VexSignalSnapshot? Signals,
|
||||
string? Summary,
|
||||
string PolicyRevisionId,
|
||||
string PolicyVersion,
|
||||
string PolicyDigest,
|
||||
IReadOnlyList<VexConsensusDecisionTelemetry> Decisions,
|
||||
VexResolveEnvelope Envelope);
|
||||
|
||||
internal sealed record VexResolveEnvelope(
|
||||
VexContentAddress Artifact,
|
||||
ResolveSignature? ContentSignature,
|
||||
VexAttestationMetadata? Attestation,
|
||||
string? AttestationEnvelope,
|
||||
ResolveSignature? AttestationSignature);
|
||||
|
||||
internal sealed record ResolveSignature(string Value, string? KeyId);
|
||||
|
||||
internal sealed record ResolveAttestation(
|
||||
VexAttestationMetadata? Metadata,
|
||||
string? Envelope,
|
||||
ResolveSignature? Signature);
|
||||
267
src/Excititor/StellaOps.Excititor.WebService/Program.cs
Normal file
267
src/Excititor/StellaOps.Excititor.WebService/Program.cs
Normal file
@@ -0,0 +1,267 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Collections.Immutable;
|
||||
using Microsoft.AspNetCore.Authentication;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Attestation.Verification;
|
||||
using StellaOps.Excititor.Attestation.Extensions;
|
||||
using StellaOps.Excititor.Attestation;
|
||||
using StellaOps.Excititor.Attestation.Transparency;
|
||||
using StellaOps.Excititor.ArtifactStores.S3.Extensions;
|
||||
using StellaOps.Excititor.Connectors.RedHat.CSAF.DependencyInjection;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Export;
|
||||
using StellaOps.Excititor.Formats.CSAF;
|
||||
using StellaOps.Excititor.Formats.CycloneDX;
|
||||
using StellaOps.Excititor.Formats.OpenVEX;
|
||||
using StellaOps.Excititor.Policy;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.WebService.Endpoints;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Aoc;
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
var configuration = builder.Configuration;
|
||||
var services = builder.Services;
|
||||
|
||||
services.AddOptions<VexMongoStorageOptions>()
|
||||
.Bind(configuration.GetSection("Excititor:Storage:Mongo"))
|
||||
.ValidateOnStart();
|
||||
|
||||
services.AddExcititorMongoStorage();
|
||||
services.AddCsafNormalizer();
|
||||
services.AddCycloneDxNormalizer();
|
||||
services.AddOpenVexNormalizer();
|
||||
services.AddSingleton<IVexSignatureVerifier, NoopVexSignatureVerifier>();
|
||||
services.AddScoped<IVexIngestOrchestrator, VexIngestOrchestrator>();
|
||||
services.AddExcititorAocGuards();
|
||||
services.AddVexExportEngine();
|
||||
services.AddVexExportCacheServices();
|
||||
services.AddVexAttestation();
|
||||
services.Configure<VexAttestationClientOptions>(configuration.GetSection("Excititor:Attestation:Client"));
|
||||
services.Configure<VexAttestationVerificationOptions>(configuration.GetSection("Excititor:Attestation:Verification"));
|
||||
services.AddVexPolicy();
|
||||
services.AddRedHatCsafConnector();
|
||||
services.Configure<MirrorDistributionOptions>(configuration.GetSection(MirrorDistributionOptions.SectionName));
|
||||
services.AddSingleton<MirrorRateLimiter>();
|
||||
|
||||
var rekorSection = configuration.GetSection("Excititor:Attestation:Rekor");
|
||||
if (rekorSection.Exists())
|
||||
{
|
||||
services.AddVexRekorClient(opts => rekorSection.Bind(opts));
|
||||
}
|
||||
|
||||
var fileSystemSection = configuration.GetSection("Excititor:Artifacts:FileSystem");
|
||||
if (fileSystemSection.Exists())
|
||||
{
|
||||
services.AddVexFileSystemArtifactStore(opts => fileSystemSection.Bind(opts));
|
||||
}
|
||||
else
|
||||
{
|
||||
services.AddVexFileSystemArtifactStore(_ => { });
|
||||
}
|
||||
|
||||
var s3Section = configuration.GetSection("Excititor:Artifacts:S3");
|
||||
if (s3Section.Exists())
|
||||
{
|
||||
services.AddVexS3ArtifactClient(opts => s3Section.GetSection("Client").Bind(opts));
|
||||
services.AddSingleton<IVexArtifactStore, S3ArtifactStore>(provider =>
|
||||
{
|
||||
var options = new S3ArtifactStoreOptions();
|
||||
s3Section.GetSection("Store").Bind(options);
|
||||
return new S3ArtifactStore(
|
||||
provider.GetRequiredService<IS3ArtifactClient>(),
|
||||
Microsoft.Extensions.Options.Options.Create(options),
|
||||
provider.GetRequiredService<Microsoft.Extensions.Logging.ILogger<S3ArtifactStore>>());
|
||||
});
|
||||
}
|
||||
|
||||
var offlineSection = configuration.GetSection("Excititor:Artifacts:OfflineBundle");
|
||||
if (offlineSection.Exists())
|
||||
{
|
||||
services.AddVexOfflineBundleArtifactStore(opts => offlineSection.Bind(opts));
|
||||
}
|
||||
|
||||
services.AddEndpointsApiExplorer();
|
||||
services.AddHealthChecks();
|
||||
services.AddSingleton(TimeProvider.System);
|
||||
services.AddMemoryCache();
|
||||
services.AddAuthentication();
|
||||
services.AddAuthorization();
|
||||
|
||||
var app = builder.Build();
|
||||
|
||||
app.UseAuthentication();
|
||||
app.UseAuthorization();
|
||||
|
||||
app.MapGet("/excititor/status", async (HttpContext context,
|
||||
IEnumerable<IVexArtifactStore> artifactStores,
|
||||
IOptions<VexMongoStorageOptions> mongoOptions,
|
||||
TimeProvider timeProvider) =>
|
||||
{
|
||||
var payload = new StatusResponse(
|
||||
timeProvider.GetUtcNow(),
|
||||
mongoOptions.Value.RawBucketName,
|
||||
mongoOptions.Value.GridFsInlineThresholdBytes,
|
||||
artifactStores.Select(store => store.GetType().Name).ToArray());
|
||||
|
||||
context.Response.ContentType = "application/json";
|
||||
await System.Text.Json.JsonSerializer.SerializeAsync(context.Response.Body, payload);
|
||||
});
|
||||
|
||||
app.MapHealthChecks("/excititor/health");
|
||||
|
||||
app.MapPost("/excititor/statements", async (
|
||||
VexStatementIngestRequest request,
|
||||
IVexClaimStore claimStore,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (request?.Statements is null || request.Statements.Count == 0)
|
||||
{
|
||||
return Results.BadRequest("At least one statement must be provided.");
|
||||
}
|
||||
|
||||
var claims = request.Statements.Select(statement => statement.ToDomainClaim());
|
||||
await claimStore.AppendAsync(claims, timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
|
||||
return Results.Accepted();
|
||||
});
|
||||
|
||||
app.MapGet("/excititor/statements/{vulnerabilityId}/{productKey}", async (
|
||||
string vulnerabilityId,
|
||||
string productKey,
|
||||
DateTimeOffset? since,
|
||||
IVexClaimStore claimStore,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(vulnerabilityId) || string.IsNullOrWhiteSpace(productKey))
|
||||
{
|
||||
return Results.BadRequest("vulnerabilityId and productKey are required.");
|
||||
}
|
||||
|
||||
var claims = await claimStore.FindAsync(vulnerabilityId.Trim(), productKey.Trim(), since, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Ok(claims);
|
||||
});
|
||||
|
||||
app.MapPost("/excititor/admin/backfill-statements", async (
|
||||
VexStatementBackfillRequest? request,
|
||||
VexStatementBackfillService backfillService,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
request ??= new VexStatementBackfillRequest();
|
||||
var result = await backfillService.RunAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
var message = FormattableString.Invariant(
|
||||
$"Backfill completed: evaluated {result.DocumentsEvaluated}, backfilled {result.DocumentsBackfilled}, claims written {result.ClaimsWritten}, skipped {result.SkippedExisting}, failures {result.NormalizationFailures}.");
|
||||
|
||||
return Results.Ok(new
|
||||
{
|
||||
message,
|
||||
summary = result
|
||||
});
|
||||
});
|
||||
|
||||
IngestEndpoints.MapIngestEndpoints(app);
|
||||
ResolveEndpoint.MapResolveEndpoint(app);
|
||||
MirrorEndpoints.MapMirrorEndpoints(app);
|
||||
|
||||
app.Run();
|
||||
|
||||
public partial class Program;
|
||||
|
||||
internal sealed record StatusResponse(DateTimeOffset UtcNow, string MongoBucket, int InlineThreshold, string[] ArtifactStores);
|
||||
|
||||
internal sealed record VexStatementIngestRequest(IReadOnlyList<VexStatementEntry> Statements);
|
||||
|
||||
internal sealed record VexStatementEntry(
|
||||
string VulnerabilityId,
|
||||
string ProviderId,
|
||||
string ProductKey,
|
||||
string? ProductName,
|
||||
string? ProductVersion,
|
||||
string? ProductPurl,
|
||||
string? ProductCpe,
|
||||
IReadOnlyList<string>? ComponentIdentifiers,
|
||||
VexClaimStatus Status,
|
||||
VexJustification? Justification,
|
||||
string? Detail,
|
||||
DateTimeOffset FirstSeen,
|
||||
DateTimeOffset LastSeen,
|
||||
VexDocumentFormat DocumentFormat,
|
||||
string DocumentDigest,
|
||||
string DocumentUri,
|
||||
string? DocumentRevision,
|
||||
VexSignatureMetadataRequest? Signature,
|
||||
VexConfidenceRequest? Confidence,
|
||||
VexSignalRequest? Signals,
|
||||
IReadOnlyDictionary<string, string>? Metadata)
|
||||
{
|
||||
public VexClaim ToDomainClaim()
|
||||
{
|
||||
var product = new VexProduct(
|
||||
ProductKey,
|
||||
ProductName,
|
||||
ProductVersion,
|
||||
ProductPurl,
|
||||
ProductCpe,
|
||||
ComponentIdentifiers ?? Array.Empty<string>());
|
||||
|
||||
if (!Uri.TryCreate(DocumentUri, UriKind.Absolute, out var uri))
|
||||
{
|
||||
throw new InvalidOperationException($"DocumentUri '{DocumentUri}' is not a valid absolute URI.");
|
||||
}
|
||||
|
||||
var document = new VexClaimDocument(
|
||||
DocumentFormat,
|
||||
DocumentDigest,
|
||||
uri,
|
||||
DocumentRevision,
|
||||
Signature?.ToDomain());
|
||||
|
||||
var additionalMetadata = Metadata is null
|
||||
? ImmutableDictionary<string, string>.Empty
|
||||
: Metadata.ToImmutableDictionary(StringComparer.Ordinal);
|
||||
|
||||
return new VexClaim(
|
||||
VulnerabilityId,
|
||||
ProviderId,
|
||||
product,
|
||||
Status,
|
||||
document,
|
||||
FirstSeen,
|
||||
LastSeen,
|
||||
Justification,
|
||||
Detail,
|
||||
Confidence?.ToDomain(),
|
||||
Signals?.ToDomain(),
|
||||
additionalMetadata);
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record VexSignatureMetadataRequest(
|
||||
string Type,
|
||||
string? Subject,
|
||||
string? Issuer,
|
||||
string? KeyId,
|
||||
DateTimeOffset? VerifiedAt,
|
||||
string? TransparencyLogReference)
|
||||
{
|
||||
public VexSignatureMetadata ToDomain()
|
||||
=> new(Type, Subject, Issuer, KeyId, VerifiedAt, TransparencyLogReference);
|
||||
}
|
||||
|
||||
internal sealed record VexConfidenceRequest(string Level, double? Score, string? Method)
|
||||
{
|
||||
public VexConfidence ToDomain() => new(Level, Score, Method);
|
||||
}
|
||||
|
||||
internal sealed record VexSignalRequest(VexSeveritySignalRequest? Severity, bool? Kev, double? Epss)
|
||||
{
|
||||
public VexSignalSnapshot ToDomain()
|
||||
=> new(Severity?.ToDomain(), Kev, Epss);
|
||||
}
|
||||
|
||||
internal sealed record VexSeveritySignalRequest(string Scheme, double? Score, string? Label, string? Vector)
|
||||
{
|
||||
public VexSeveritySignal ToDomain() => new(Scheme, Score, Label, Vector);
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Excititor.WebService.Tests")]
|
||||
@@ -0,0 +1,57 @@
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Services;
|
||||
|
||||
internal sealed class MirrorRateLimiter
|
||||
{
|
||||
private readonly IMemoryCache _cache;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private static readonly TimeSpan Window = TimeSpan.FromHours(1);
|
||||
|
||||
public MirrorRateLimiter(IMemoryCache cache, TimeProvider timeProvider)
|
||||
{
|
||||
_cache = cache ?? throw new ArgumentNullException(nameof(cache));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
public bool TryAcquire(string domainId, string scope, int limit, out TimeSpan? retryAfter)
|
||||
{
|
||||
retryAfter = null;
|
||||
|
||||
if (limit <= 0 || limit == int.MaxValue)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var key = CreateKey(domainId, scope);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
var counter = _cache.Get<Counter>(key);
|
||||
if (counter is null || now - counter.WindowStart >= Window)
|
||||
{
|
||||
counter = new Counter(now, 0);
|
||||
}
|
||||
|
||||
if (counter.Count >= limit)
|
||||
{
|
||||
var windowEnd = counter.WindowStart + Window;
|
||||
retryAfter = windowEnd > now ? windowEnd - now : TimeSpan.Zero;
|
||||
return false;
|
||||
}
|
||||
|
||||
counter = counter with { Count = counter.Count + 1 };
|
||||
var absoluteExpiration = counter.WindowStart + Window;
|
||||
_cache.Set(key, counter, absoluteExpiration);
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string CreateKey(string domainId, string scope)
|
||||
=> string.Create(domainId.Length + scope.Length + 1, (domainId, scope), static (span, state) =>
|
||||
{
|
||||
state.domainId.AsSpan().CopyTo(span);
|
||||
span[state.domainId.Length] = '|';
|
||||
state.scope.AsSpan().CopyTo(span[(state.domainId.Length + 1)..]);
|
||||
});
|
||||
|
||||
private sealed record Counter(DateTimeOffset WindowStart, int Count);
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
using System.Linq;
|
||||
using System.Security.Claims;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Services;
|
||||
|
||||
internal static class ScopeAuthorization
|
||||
{
|
||||
public static IResult? RequireScope(HttpContext context, string scope)
|
||||
{
|
||||
if (context is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(context));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(scope))
|
||||
{
|
||||
throw new ArgumentException("Scope must be provided.", nameof(scope));
|
||||
}
|
||||
|
||||
var user = context.User;
|
||||
if (user?.Identity?.IsAuthenticated is not true)
|
||||
{
|
||||
return Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!HasScope(user, scope))
|
||||
{
|
||||
return Results.Forbid();
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool HasScope(ClaimsPrincipal user, string requiredScope)
|
||||
{
|
||||
var comparison = StringComparer.OrdinalIgnoreCase;
|
||||
foreach (var claim in user.FindAll("scope").Concat(user.FindAll("scp")))
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(claim.Value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var scopes = claim.Value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
if (scopes.Any(scope => comparison.Equals(scope, requiredScope)))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,584 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Services;
|
||||
|
||||
internal interface IVexIngestOrchestrator
|
||||
{
|
||||
Task<InitSummary> InitializeAsync(IngestInitOptions options, CancellationToken cancellationToken);
|
||||
|
||||
Task<IngestRunSummary> RunAsync(IngestRunOptions options, CancellationToken cancellationToken);
|
||||
|
||||
Task<IngestRunSummary> ResumeAsync(IngestResumeOptions options, CancellationToken cancellationToken);
|
||||
|
||||
Task<ReconcileSummary> ReconcileAsync(ReconcileOptions options, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
|
||||
{
|
||||
private readonly IServiceProvider _serviceProvider;
|
||||
private readonly IReadOnlyDictionary<string, IVexConnector> _connectors;
|
||||
private readonly IVexRawStore _rawStore;
|
||||
private readonly IVexClaimStore _claimStore;
|
||||
private readonly IVexProviderStore _providerStore;
|
||||
private readonly IVexConnectorStateRepository _stateRepository;
|
||||
private readonly IVexNormalizerRouter _normalizerRouter;
|
||||
private readonly IVexSignatureVerifier _signatureVerifier;
|
||||
private readonly IVexMongoSessionProvider _sessionProvider;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<VexIngestOrchestrator> _logger;
|
||||
|
||||
public VexIngestOrchestrator(
|
||||
IServiceProvider serviceProvider,
|
||||
IEnumerable<IVexConnector> connectors,
|
||||
IVexRawStore rawStore,
|
||||
IVexClaimStore claimStore,
|
||||
IVexProviderStore providerStore,
|
||||
IVexConnectorStateRepository stateRepository,
|
||||
IVexNormalizerRouter normalizerRouter,
|
||||
IVexSignatureVerifier signatureVerifier,
|
||||
IVexMongoSessionProvider sessionProvider,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<VexIngestOrchestrator> logger)
|
||||
{
|
||||
_serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
|
||||
_rawStore = rawStore ?? throw new ArgumentNullException(nameof(rawStore));
|
||||
_claimStore = claimStore ?? throw new ArgumentNullException(nameof(claimStore));
|
||||
_providerStore = providerStore ?? throw new ArgumentNullException(nameof(providerStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_normalizerRouter = normalizerRouter ?? throw new ArgumentNullException(nameof(normalizerRouter));
|
||||
_signatureVerifier = signatureVerifier ?? throw new ArgumentNullException(nameof(signatureVerifier));
|
||||
_sessionProvider = sessionProvider ?? throw new ArgumentNullException(nameof(sessionProvider));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
|
||||
if (connectors is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(connectors));
|
||||
}
|
||||
|
||||
_connectors = connectors
|
||||
.GroupBy(connector => connector.Id, StringComparer.OrdinalIgnoreCase)
|
||||
.ToDictionary(group => group.Key, group => group.First(), StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public async Task<InitSummary> InitializeAsync(IngestInitOptions options, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
var runId = Guid.NewGuid();
|
||||
var startedAt = _timeProvider.GetUtcNow();
|
||||
var results = ImmutableArray.CreateBuilder<InitProviderResult>();
|
||||
|
||||
var session = await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var (handles, missing) = ResolveConnectors(options.Providers);
|
||||
foreach (var providerId in missing)
|
||||
{
|
||||
results.Add(new InitProviderResult(providerId, providerId, "missing", TimeSpan.Zero, "Provider connector is not registered."));
|
||||
}
|
||||
|
||||
foreach (var handle in handles)
|
||||
{
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
await ValidateConnectorAsync(handle, cancellationToken).ConfigureAwait(false);
|
||||
await EnsureProviderRegistrationAsync(handle.Descriptor, session, cancellationToken).ConfigureAwait(false);
|
||||
stopwatch.Stop();
|
||||
|
||||
results.Add(new InitProviderResult(
|
||||
handle.Descriptor.Id,
|
||||
handle.Descriptor.DisplayName,
|
||||
"succeeded",
|
||||
stopwatch.Elapsed,
|
||||
Error: null));
|
||||
|
||||
_logger.LogInformation("Excititor init validated provider {ProviderId} in {Duration}ms.", handle.Descriptor.Id, stopwatch.Elapsed.TotalMilliseconds);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
results.Add(new InitProviderResult(
|
||||
handle.Descriptor.Id,
|
||||
handle.Descriptor.DisplayName,
|
||||
"cancelled",
|
||||
stopwatch.Elapsed,
|
||||
"Operation cancelled."));
|
||||
_logger.LogWarning("Excititor init cancelled for provider {ProviderId}.", handle.Descriptor.Id);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
results.Add(new InitProviderResult(
|
||||
handle.Descriptor.Id,
|
||||
handle.Descriptor.DisplayName,
|
||||
"failed",
|
||||
stopwatch.Elapsed,
|
||||
ex.Message));
|
||||
_logger.LogError(ex, "Excititor init failed for provider {ProviderId}: {Message}", handle.Descriptor.Id, ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
var completedAt = _timeProvider.GetUtcNow();
|
||||
return new InitSummary(runId, startedAt, completedAt, results.ToImmutable());
|
||||
}
|
||||
|
||||
public async Task<IngestRunSummary> RunAsync(IngestRunOptions options, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
var runId = Guid.NewGuid();
|
||||
var startedAt = _timeProvider.GetUtcNow();
|
||||
var since = ResolveSince(options.Since, options.Window, startedAt);
|
||||
var results = ImmutableArray.CreateBuilder<ProviderRunResult>();
|
||||
var session = await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var (handles, missing) = ResolveConnectors(options.Providers);
|
||||
foreach (var providerId in missing)
|
||||
{
|
||||
results.Add(ProviderRunResult.Missing(providerId, since));
|
||||
}
|
||||
|
||||
foreach (var handle in handles)
|
||||
{
|
||||
var result = await ExecuteRunAsync(handle, since, options.Force, session, cancellationToken).ConfigureAwait(false);
|
||||
results.Add(result);
|
||||
}
|
||||
|
||||
var completedAt = _timeProvider.GetUtcNow();
|
||||
return new IngestRunSummary(runId, startedAt, completedAt, results.ToImmutable());
|
||||
}
|
||||
|
||||
public async Task<IngestRunSummary> ResumeAsync(IngestResumeOptions options, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
var runId = Guid.NewGuid();
|
||||
var startedAt = _timeProvider.GetUtcNow();
|
||||
var results = ImmutableArray.CreateBuilder<ProviderRunResult>();
|
||||
var session = await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var (handles, missing) = ResolveConnectors(options.Providers);
|
||||
foreach (var providerId in missing)
|
||||
{
|
||||
results.Add(ProviderRunResult.Missing(providerId, since: null));
|
||||
}
|
||||
|
||||
foreach (var handle in handles)
|
||||
{
|
||||
var since = await ResolveResumeSinceAsync(handle.Descriptor.Id, options.Checkpoint, session, cancellationToken).ConfigureAwait(false);
|
||||
var result = await ExecuteRunAsync(handle, since, force: false, session, cancellationToken).ConfigureAwait(false);
|
||||
results.Add(result);
|
||||
}
|
||||
|
||||
var completedAt = _timeProvider.GetUtcNow();
|
||||
return new IngestRunSummary(runId, startedAt, completedAt, results.ToImmutable());
|
||||
}
|
||||
|
||||
public async Task<ReconcileSummary> ReconcileAsync(ReconcileOptions options, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
var runId = Guid.NewGuid();
|
||||
var startedAt = _timeProvider.GetUtcNow();
|
||||
var threshold = options.MaxAge is null ? (DateTimeOffset?)null : startedAt - options.MaxAge.Value;
|
||||
var results = ImmutableArray.CreateBuilder<ReconcileProviderResult>();
|
||||
var session = await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var (handles, missing) = ResolveConnectors(options.Providers);
|
||||
foreach (var providerId in missing)
|
||||
{
|
||||
results.Add(new ReconcileProviderResult(providerId, "missing", "missing", null, threshold, 0, 0, "Provider connector is not registered."));
|
||||
}
|
||||
|
||||
foreach (var handle in handles)
|
||||
{
|
||||
try
|
||||
{
|
||||
var state = await _stateRepository.GetAsync(handle.Descriptor.Id, cancellationToken, session).ConfigureAwait(false);
|
||||
var lastUpdated = state?.LastUpdated;
|
||||
var stale = threshold.HasValue && (lastUpdated is null || lastUpdated < threshold.Value);
|
||||
|
||||
if (stale || state is null)
|
||||
{
|
||||
var since = stale ? threshold : lastUpdated;
|
||||
var result = await ExecuteRunAsync(handle, since, force: false, session, cancellationToken).ConfigureAwait(false);
|
||||
results.Add(new ReconcileProviderResult(
|
||||
handle.Descriptor.Id,
|
||||
result.Status,
|
||||
"reconciled",
|
||||
result.LastUpdated ?? result.CompletedAt,
|
||||
threshold,
|
||||
result.Documents,
|
||||
result.Claims,
|
||||
result.Error));
|
||||
}
|
||||
else
|
||||
{
|
||||
results.Add(new ReconcileProviderResult(
|
||||
handle.Descriptor.Id,
|
||||
"succeeded",
|
||||
"skipped",
|
||||
lastUpdated,
|
||||
threshold,
|
||||
Documents: 0,
|
||||
Claims: 0,
|
||||
Error: null));
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
results.Add(new ReconcileProviderResult(
|
||||
handle.Descriptor.Id,
|
||||
"cancelled",
|
||||
"cancelled",
|
||||
null,
|
||||
threshold,
|
||||
0,
|
||||
0,
|
||||
"Operation cancelled."));
|
||||
_logger.LogWarning("Excititor reconcile cancelled for provider {ProviderId}.", handle.Descriptor.Id);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
results.Add(new ReconcileProviderResult(
|
||||
handle.Descriptor.Id,
|
||||
"failed",
|
||||
"failed",
|
||||
null,
|
||||
threshold,
|
||||
0,
|
||||
0,
|
||||
ex.Message));
|
||||
_logger.LogError(ex, "Excititor reconcile failed for provider {ProviderId}: {Message}", handle.Descriptor.Id, ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
var completedAt = _timeProvider.GetUtcNow();
|
||||
return new ReconcileSummary(runId, startedAt, completedAt, results.ToImmutable());
|
||||
}
|
||||
|
||||
private async Task ValidateConnectorAsync(ConnectorHandle handle, CancellationToken cancellationToken)
|
||||
{
|
||||
await handle.Connector.ValidateAsync(VexConnectorSettings.Empty, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task EnsureProviderRegistrationAsync(VexConnectorDescriptor descriptor, IClientSessionHandle session, CancellationToken cancellationToken)
|
||||
{
|
||||
var existing = await _providerStore.FindAsync(descriptor.Id, cancellationToken, session).ConfigureAwait(false);
|
||||
if (existing is not null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var provider = new VexProvider(descriptor.Id, descriptor.DisplayName, descriptor.Kind);
|
||||
await _providerStore.SaveAsync(provider, cancellationToken, session).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<ProviderRunResult> ExecuteRunAsync(
|
||||
ConnectorHandle handle,
|
||||
DateTimeOffset? since,
|
||||
bool force,
|
||||
IClientSessionHandle session,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var providerId = handle.Descriptor.Id;
|
||||
var startedAt = _timeProvider.GetUtcNow();
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
|
||||
try
|
||||
{
|
||||
await ValidateConnectorAsync(handle, cancellationToken).ConfigureAwait(false);
|
||||
await EnsureProviderRegistrationAsync(handle.Descriptor, session, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (force)
|
||||
{
|
||||
var resetState = new VexConnectorState(providerId, null, ImmutableArray<string>.Empty);
|
||||
await _stateRepository.SaveAsync(resetState, cancellationToken, session).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var stateBeforeRun = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false);
|
||||
var resumeTokens = stateBeforeRun?.ResumeTokens ?? ImmutableDictionary<string, string>.Empty;
|
||||
|
||||
var context = new VexConnectorContext(
|
||||
since,
|
||||
VexConnectorSettings.Empty,
|
||||
_rawStore,
|
||||
_signatureVerifier,
|
||||
_normalizerRouter,
|
||||
_serviceProvider,
|
||||
resumeTokens);
|
||||
|
||||
var documents = 0;
|
||||
var claims = 0;
|
||||
string? lastDigest = null;
|
||||
|
||||
await foreach (var document in handle.Connector.FetchAsync(context, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
documents++;
|
||||
lastDigest = document.Digest;
|
||||
|
||||
var batch = await _normalizerRouter.NormalizeAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
if (!batch.Claims.IsDefaultOrEmpty && batch.Claims.Length > 0)
|
||||
{
|
||||
claims += batch.Claims.Length;
|
||||
await _claimStore.AppendAsync(batch.Claims, _timeProvider.GetUtcNow(), cancellationToken, session).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
stopwatch.Stop();
|
||||
var completedAt = _timeProvider.GetUtcNow();
|
||||
var stateAfterRun = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false);
|
||||
|
||||
var checkpoint = stateAfterRun?.DocumentDigests.IsDefaultOrEmpty == false
|
||||
? stateAfterRun.DocumentDigests[^1]
|
||||
: lastDigest;
|
||||
|
||||
var result = new ProviderRunResult(
|
||||
providerId,
|
||||
"succeeded",
|
||||
documents,
|
||||
claims,
|
||||
startedAt,
|
||||
completedAt,
|
||||
stopwatch.Elapsed,
|
||||
lastDigest,
|
||||
stateAfterRun?.LastUpdated,
|
||||
checkpoint,
|
||||
null,
|
||||
since);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Excititor ingest provider {ProviderId} completed: documents={Documents} claims={Claims} since={Since} duration={Duration}ms",
|
||||
providerId,
|
||||
documents,
|
||||
claims,
|
||||
since?.ToString("O", CultureInfo.InvariantCulture),
|
||||
result.Duration.TotalMilliseconds);
|
||||
|
||||
return result;
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
var cancelledAt = _timeProvider.GetUtcNow();
|
||||
_logger.LogWarning("Excititor ingest provider {ProviderId} cancelled.", providerId);
|
||||
return new ProviderRunResult(
|
||||
providerId,
|
||||
"cancelled",
|
||||
0,
|
||||
0,
|
||||
startedAt,
|
||||
cancelledAt,
|
||||
stopwatch.Elapsed,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
"Operation cancelled.",
|
||||
since);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
var failedAt = _timeProvider.GetUtcNow();
|
||||
_logger.LogError(ex, "Excititor ingest provider {ProviderId} failed: {Message}", providerId, ex.Message);
|
||||
return new ProviderRunResult(
|
||||
providerId,
|
||||
"failed",
|
||||
0,
|
||||
0,
|
||||
startedAt,
|
||||
failedAt,
|
||||
stopwatch.Elapsed,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
ex.Message,
|
||||
since);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<DateTimeOffset?> ResolveResumeSinceAsync(string providerId, string? checkpoint, IClientSessionHandle session, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(checkpoint))
|
||||
{
|
||||
if (DateTimeOffset.TryParse(
|
||||
checkpoint.Trim(),
|
||||
CultureInfo.InvariantCulture,
|
||||
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
|
||||
out var parsed))
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
|
||||
var digest = checkpoint.Trim();
|
||||
var document = await _rawStore.FindByDigestAsync(digest, cancellationToken, session).ConfigureAwait(false);
|
||||
if (document is not null)
|
||||
{
|
||||
return document.RetrievedAt;
|
||||
}
|
||||
}
|
||||
|
||||
var state = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false);
|
||||
return state?.LastUpdated;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ResolveSince(DateTimeOffset? since, TimeSpan? window, DateTimeOffset reference)
|
||||
{
|
||||
if (since.HasValue)
|
||||
{
|
||||
return since.Value;
|
||||
}
|
||||
|
||||
if (window is { } duration && duration > TimeSpan.Zero)
|
||||
{
|
||||
var candidate = reference - duration;
|
||||
return candidate < DateTimeOffset.MinValue ? DateTimeOffset.MinValue : candidate;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private (IReadOnlyList<ConnectorHandle> Handles, ImmutableArray<string> Missing) ResolveConnectors(ImmutableArray<string> requestedProviders)
|
||||
{
|
||||
var handles = new List<ConnectorHandle>();
|
||||
var missing = ImmutableArray.CreateBuilder<string>();
|
||||
|
||||
if (requestedProviders.IsDefaultOrEmpty || requestedProviders.Length == 0)
|
||||
{
|
||||
foreach (var connector in _connectors.Values.OrderBy(static x => x.Id, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
handles.Add(new ConnectorHandle(connector, CreateDescriptor(connector)));
|
||||
}
|
||||
|
||||
return (handles, missing.ToImmutable());
|
||||
}
|
||||
|
||||
foreach (var providerId in requestedProviders)
|
||||
{
|
||||
if (_connectors.TryGetValue(providerId, out var connector))
|
||||
{
|
||||
handles.Add(new ConnectorHandle(connector, CreateDescriptor(connector)));
|
||||
}
|
||||
else
|
||||
{
|
||||
missing.Add(providerId);
|
||||
}
|
||||
}
|
||||
|
||||
return (handles, missing.ToImmutable());
|
||||
}
|
||||
|
||||
private static VexConnectorDescriptor CreateDescriptor(IVexConnector connector)
|
||||
=> connector switch
|
||||
{
|
||||
VexConnectorBase baseConnector => baseConnector.Descriptor,
|
||||
_ => new VexConnectorDescriptor(connector.Id, connector.Kind, connector.Id)
|
||||
};
|
||||
|
||||
private sealed record ConnectorHandle(IVexConnector Connector, VexConnectorDescriptor Descriptor);
|
||||
}
|
||||
|
||||
internal sealed record IngestInitOptions(
|
||||
ImmutableArray<string> Providers,
|
||||
bool Resume);
|
||||
|
||||
internal sealed record IngestRunOptions(
|
||||
ImmutableArray<string> Providers,
|
||||
DateTimeOffset? Since,
|
||||
TimeSpan? Window,
|
||||
bool Force);
|
||||
|
||||
internal sealed record IngestResumeOptions(
|
||||
ImmutableArray<string> Providers,
|
||||
string? Checkpoint);
|
||||
|
||||
internal sealed record ReconcileOptions(
|
||||
ImmutableArray<string> Providers,
|
||||
TimeSpan? MaxAge);
|
||||
|
||||
internal sealed record InitSummary(
|
||||
Guid RunId,
|
||||
DateTimeOffset StartedAt,
|
||||
DateTimeOffset CompletedAt,
|
||||
ImmutableArray<InitProviderResult> Providers)
|
||||
{
|
||||
public int ProviderCount => Providers.Length;
|
||||
public int SuccessCount => Providers.Count(result => string.Equals(result.Status, "succeeded", StringComparison.OrdinalIgnoreCase));
|
||||
public int FailureCount => Providers.Count(result => string.Equals(result.Status, "failed", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
internal sealed record InitProviderResult(
|
||||
string ProviderId,
|
||||
string DisplayName,
|
||||
string Status,
|
||||
TimeSpan Duration,
|
||||
string? Error);
|
||||
|
||||
internal sealed record IngestRunSummary(
|
||||
Guid RunId,
|
||||
DateTimeOffset StartedAt,
|
||||
DateTimeOffset CompletedAt,
|
||||
ImmutableArray<ProviderRunResult> Providers)
|
||||
{
|
||||
public int ProviderCount => Providers.Length;
|
||||
|
||||
public int SuccessCount => Providers.Count(provider => string.Equals(provider.Status, "succeeded", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
public int FailureCount => Providers.Count(provider => string.Equals(provider.Status, "failed", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
public TimeSpan Duration => CompletedAt - StartedAt;
|
||||
}
|
||||
|
||||
internal sealed record ProviderRunResult(
|
||||
string ProviderId,
|
||||
string Status,
|
||||
int Documents,
|
||||
int Claims,
|
||||
DateTimeOffset StartedAt,
|
||||
DateTimeOffset CompletedAt,
|
||||
TimeSpan Duration,
|
||||
string? LastDigest,
|
||||
DateTimeOffset? LastUpdated,
|
||||
string? Checkpoint,
|
||||
string? Error,
|
||||
DateTimeOffset? Since)
|
||||
{
|
||||
public static ProviderRunResult Missing(string providerId, DateTimeOffset? since)
|
||||
=> new(providerId, "missing", 0, 0, DateTimeOffset.MinValue, DateTimeOffset.MinValue, TimeSpan.Zero, null, null, null, "Provider connector is not registered.", since);
|
||||
}
|
||||
|
||||
internal sealed record ReconcileSummary(
|
||||
Guid RunId,
|
||||
DateTimeOffset StartedAt,
|
||||
DateTimeOffset CompletedAt,
|
||||
ImmutableArray<ReconcileProviderResult> Providers)
|
||||
{
|
||||
public int ProviderCount => Providers.Length;
|
||||
|
||||
public int ReconciledCount => Providers.Count(result => string.Equals(result.Action, "reconciled", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
public int SkippedCount => Providers.Count(result => string.Equals(result.Action, "skipped", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
public int FailureCount => Providers.Count(result => string.Equals(result.Status, "failed", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
public TimeSpan Duration => CompletedAt - StartedAt;
|
||||
}
|
||||
|
||||
internal sealed record ReconcileProviderResult(
|
||||
string ProviderId,
|
||||
string Status,
|
||||
string Action,
|
||||
DateTimeOffset? LastUpdated,
|
||||
DateTimeOffset? Threshold,
|
||||
int Documents,
|
||||
int Claims,
|
||||
string? Error);
|
||||
@@ -0,0 +1,23 @@
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Export/StellaOps.Excititor.Export.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Policy/StellaOps.Excititor.Policy.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Attestation/StellaOps.Excititor.Attestation.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.ArtifactStores.S3/StellaOps.Excititor.ArtifactStores.S3.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Connectors.RedHat.CSAF/StellaOps.Excititor.Connectors.RedHat.CSAF.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Formats.CSAF/StellaOps.Excititor.Formats.CSAF.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Formats.CycloneDX/StellaOps.Excititor.Formats.CycloneDX.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Formats.OpenVEX/StellaOps.Excititor.Formats.OpenVEX.csproj" />
|
||||
<ProjectReference Include="../../Aoc/__Libraries/StellaOps.Aoc/StellaOps.Aoc.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
94
src/Excititor/StellaOps.Excititor.WebService/TASKS.md
Normal file
94
src/Excititor/StellaOps.Excititor.WebService/TASKS.md
Normal file
@@ -0,0 +1,94 @@
|
||||
# TASKS — Epic 1: Aggregation-Only Contract
|
||||
> **AOC Reminder:** Excititor WebService publishes raw statements/linksets only; derived precedence/severity belongs to Policy overlays.
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|---|---|---|---|---|
|
||||
| EXCITITOR-WEB-AOC-19-001 `Raw VEX ingestion APIs` | TODO | Excititor WebService Guild | EXCITITOR-CORE-AOC-19-001, EXCITITOR-STORE-AOC-19-001 | Implement `POST /ingest/vex`, `GET /vex/raw*`, and `POST /aoc/verify` endpoints. Enforce Authority scopes, tenant injection, and guard pipeline to ensure only immutable VEX facts are persisted. |
|
||||
> Docs alignment (2025-10-26): See AOC reference §4–5 and authority scopes doc for required tokens/behaviour.
|
||||
| EXCITITOR-WEB-AOC-19-002 `AOC observability + metrics` | TODO | Excititor WebService Guild, Observability Guild | EXCITITOR-WEB-AOC-19-001 | Export metrics (`ingestion_write_total`, `aoc_violation_total`, signature verification counters) and tracing spans matching Conseiller naming. Ensure structured logging includes tenant, source vendor, upstream id, and content hash. |
|
||||
> Docs alignment (2025-10-26): Metrics/traces/log schema in `docs/observability/observability.md`.
|
||||
| EXCITITOR-WEB-AOC-19-003 `Guard + schema test harness` | TODO | QA Guild | EXCITITOR-WEB-AOC-19-001 | Add unit/integration tests for schema validation, forbidden field rejection (`ERR_AOC_001/006/007`), and supersedes behavior using CycloneDX-VEX & CSAF fixtures with deterministic expectations. |
|
||||
> Docs alignment (2025-10-26): Error codes + CLI verification in `docs/cli/cli-reference.md`.
|
||||
| EXCITITOR-WEB-AOC-19-004 `Batch ingest validation` | TODO | Excititor WebService Guild, QA Guild | EXCITITOR-WEB-AOC-19-003, EXCITITOR-CORE-AOC-19-002 | Build large fixture ingest covering mixed VEX statuses, verifying raw storage parity, metrics, and CLI `aoc verify` compatibility. Document load test/runbook updates. |
|
||||
> Docs alignment (2025-10-26): Offline/air-gap workflows captured in `docs/deploy/containers.md` §5.
|
||||
|
||||
## Policy Engine v2
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-POLICY-20-001 `Policy selection endpoints` | TODO | Excititor WebService Guild | WEB-POLICY-20-001, EXCITITOR-CORE-AOC-19-004 | Provide VEX lookup APIs supporting PURL/advisory batching, scope filtering, and tenant enforcement with deterministic ordering + pagination. |
|
||||
|
||||
## StellaOps Console (Sprint 23)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-CONSOLE-23-001 `VEX aggregation views` | TODO | Excititor WebService Guild, BE-Base Platform Guild | EXCITITOR-LNM-21-201, EXCITITOR-LNM-21-202 | Expose `/console/vex` endpoints returning grouped VEX statements per advisory/component with status chips, justification metadata, precedence trace pointers, and tenant-scoped filters for Console explorer. |
|
||||
| EXCITITOR-CONSOLE-23-002 `Dashboard VEX deltas` | TODO | Excititor WebService Guild | EXCITITOR-CONSOLE-23-001, EXCITITOR-LNM-21-203 | Provide aggregated counts for VEX overrides (new, not_affected, revoked) powering Console dashboard + live status ticker; emit metrics for policy explain integration. |
|
||||
| EXCITITOR-CONSOLE-23-003 `VEX search helpers` | TODO | Excititor WebService Guild | EXCITITOR-CONSOLE-23-001 | Deliver rapid lookup endpoints of VEX by advisory/component for Console global search; ensure response includes provenance and precedence context; include caching and RBAC. |
|
||||
|
||||
## Graph Explorer v1
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
|
||||
## Link-Not-Merge v1
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-LNM-21-201 `Observation APIs` | TODO | Excititor WebService Guild, BE-Base Platform Guild | EXCITITOR-LNM-21-001 | Add VEX observation read endpoints with filters, pagination, RBAC, and tenant scoping. |
|
||||
| EXCITITOR-LNM-21-202 `Linkset APIs` | TODO | Excititor WebService Guild | EXCITITOR-LNM-21-002, EXCITITOR-LNM-21-003 | Implement linkset read/export/evidence endpoints returning correlation/conflict payloads and map errors to `ERR_AGG_*`. |
|
||||
| EXCITITOR-LNM-21-203 `Event publishing` | TODO | Excititor WebService Guild, Platform Events Guild | EXCITITOR-LNM-21-005 | Publish `vex.linkset.updated` events, document schema, and ensure idempotent delivery. |
|
||||
|
||||
## Graph & Vuln Explorer v1
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-GRAPH-24-101 `VEX summary API` | TODO | Excititor WebService Guild | EXCITITOR-GRAPH-24-001 | Provide endpoints delivering VEX status summaries per component/asset for Vuln Explorer integration. |
|
||||
| EXCITITOR-GRAPH-24-102 `Evidence batch API` | TODO | Excititor WebService Guild | EXCITITOR-LNM-21-201 | Add batch VEX observation retrieval optimized for Graph overlays/tooltips. |
|
||||
|
||||
## VEX Lens (Sprint 30)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-VEXLENS-30-001 `VEX evidence enrichers` | TODO | Excititor WebService Guild, VEX Lens Guild | EXCITITOR-VULN-29-001, VEXLENS-30-005 | Include issuer hints, signatures, and product trees in evidence payloads for VEX Lens; Label: VEX-Lens. |
|
||||
|
||||
## Vulnerability Explorer (Sprint 29)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-VULN-29-001 `VEX key canonicalization` | TODO | Excititor WebService Guild | EXCITITOR-LNM-21-001 | Canonicalize (lossless) VEX advisory/product keys (map to `advisory_key`, capture product scopes); expose original sources in `links[]`; AOC-compliant: no merge, no derived fields, no suppression; backfill existing records. |
|
||||
| EXCITITOR-VULN-29-002 `Evidence retrieval` | TODO | Excititor WebService Guild | EXCITITOR-VULN-29-001, VULN-API-29-003 | Provide `/vuln/evidence/vex/{advisory_key}` returning raw VEX statements filtered by tenant/product scope for Explorer evidence tabs. |
|
||||
| EXCITITOR-VULN-29-004 `Observability` | TODO | Excititor WebService Guild, Observability Guild | EXCITITOR-VULN-29-001 | Add metrics/logs for VEX normalization, suppression scopes, withdrawn statements; emit events consumed by Vuln Explorer resolver. |
|
||||
|
||||
## Advisory AI (Sprint 31)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-AIAI-31-001 `Justification enrichment` | TODO | Excititor WebService Guild | EXCITITOR-VULN-29-001 | Expose normalized VEX justifications, product trees, and paragraph anchors for Advisory AI conflict explanations. |
|
||||
| EXCITITOR-AIAI-31-002 `VEX chunk API` | TODO | Excititor WebService Guild | EXCITITOR-AIAI-31-001, VEXLENS-30-006 | Provide `/vex/evidence/chunks` endpoint returning tenant-scoped VEX statements with signature metadata and scope scores for RAG. |
|
||||
| EXCITITOR-AIAI-31-003 `Telemetry` | TODO | Excititor WebService Guild, Observability Guild | EXCITITOR-AIAI-31-001 | Emit metrics/logs for VEX chunk usage, signature verification failures, and guardrail triggers. |
|
||||
|
||||
## Observability & Forensics (Epic 15)
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-WEB-OBS-50-001 `Telemetry adoption` | TODO | Excititor WebService Guild | TELEMETRY-OBS-50-001, EXCITITOR-OBS-50-001 | Adopt telemetry core for VEX APIs, ensure responses include trace IDs & correlation headers, and update structured logging for read endpoints. |
|
||||
| EXCITITOR-WEB-OBS-51-001 `Observability health endpoints` | TODO | Excititor WebService Guild | EXCITITOR-WEB-OBS-50-001, WEB-OBS-51-001 | Implement `/obs/excititor/health` summarizing ingest/link SLOs, signature failure counts, and conflict trends for Console dashboards. |
|
||||
| EXCITITOR-WEB-OBS-52-001 `Timeline streaming` | TODO | Excititor WebService Guild | EXCITITOR-WEB-OBS-50-001, TIMELINE-OBS-52-003 | Provide SSE bridge for VEX timeline events with tenant filters, pagination, and guardrails. |
|
||||
| EXCITITOR-WEB-OBS-53-001 `Evidence APIs` | TODO | Excititor WebService Guild, Evidence Locker Guild | EXCITITOR-OBS-53-001, EVID-OBS-53-003 | Expose `/evidence/vex/*` endpoints that fetch locker bundles, enforce scopes, and surface verification metadata. |
|
||||
| EXCITITOR-WEB-OBS-54-001 `Attestation APIs` | TODO | Excititor WebService Guild | EXCITITOR-OBS-54-001, PROV-OBS-54-001 | Add `/attestations/vex/*` endpoints returning DSSE verification state, builder identity, and chain-of-custody links. |
|
||||
| EXCITITOR-WEB-OBS-55-001 `Incident mode toggles` | TODO | Excititor WebService Guild, DevOps Guild | EXCITITOR-OBS-55-001, WEB-OBS-55-001 | Provide incident mode API for VEX pipelines with activation audit logs and retention override previews. |
|
||||
|
||||
## Air-Gapped Mode (Epic 16)
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-WEB-AIRGAP-56-001 | TODO | Excititor WebService Guild | AIRGAP-IMP-58-001, EXCITITOR-AIRGAP-56-001 | Support mirror bundle registration via APIs, expose bundle provenance in VEX responses, and block external connectors in sealed mode. |
|
||||
| EXCITITOR-WEB-AIRGAP-56-002 | TODO | Excititor WebService Guild, AirGap Time Guild | EXCITITOR-WEB-AIRGAP-56-001, AIRGAP-TIME-58-001 | Return VEX staleness metrics and time anchor info in API responses for Console/CLI use. |
|
||||
| EXCITITOR-WEB-AIRGAP-57-001 | TODO | Excititor WebService Guild, AirGap Policy Guild | AIRGAP-POL-56-001 | Map sealed-mode violations to standardized error payload with remediation guidance. |
|
||||
| EXCITITOR-WEB-AIRGAP-58-001 | TODO | Excititor WebService Guild, AirGap Importer Guild | EXCITITOR-WEB-AIRGAP-56-001, TIMELINE-OBS-53-001 | Emit timeline events for VEX bundle imports with bundle ID, scope, and actor metadata. |
|
||||
|
||||
## SDKs & OpenAPI (Epic 17)
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-WEB-OAS-61-001 | TODO | Excititor WebService Guild | OAS-61-001 | Implement `/.well-known/openapi` discovery endpoint with spec version metadata. |
|
||||
| EXCITITOR-WEB-OAS-61-002 | TODO | Excititor WebService Guild | APIGOV-61-001 | Standardize error envelope responses and update controller/unit tests. |
|
||||
| EXCITITOR-WEB-OAS-62-001 | TODO | Excititor WebService Guild | EXCITITOR-OAS-61-002 | Add curated examples for VEX observation/linkset endpoints and ensure portal displays them. |
|
||||
| EXCITITOR-WEB-OAS-63-001 | TODO | Excititor WebService Guild, API Governance Guild | APIGOV-63-001 | Emit deprecation headers and update docs for retiring VEX APIs. |
|
||||
Reference in New Issue
Block a user