stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

up

Browse Source
This commit is contained in:
StellaOps Bot
2025-12-09 00:20:52 +02:00
parent 3d01bf9edc
commit bc0762e97d
261 changed files with 14033 additions and 4427 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/Excititor/StellaOps.Excititor.WebService/Endpoints/AttestationEndpoints.cs
View File

@@ -9,7 +9,7 @@ using Microsoft.Extensions.Options;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Excititor.WebService.Contracts;
using StellaOps.Excititor.WebService.Services;
@@ -27,7 +27,7 @@ public static class AttestationEndpoints
// GET /attestations/vex/list - List attestations
app.MapGet("/attestations/vex/list", async (
HttpContext context,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IMongoDatabase database,
TimeProvider timeProvider,
[FromQuery] int? limit,
@@ -102,7 +102,7 @@ public static class AttestationEndpoints
app.MapGet("/attestations/vex/{attestationId}", async (
HttpContext context,
string attestationId,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IVexAttestationLinkStore attestationStore,
TimeProvider timeProvider,
CancellationToken cancellationToken) =>
@@ -209,7 +209,7 @@ public static class AttestationEndpoints
// GET /attestations/vex/lookup - Lookup attestations by linkset or observation
app.MapGet("/attestations/vex/lookup", async (
HttpContext context,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IMongoDatabase database,
TimeProvider timeProvider,
[FromQuery] string? linksetId,
@@ -283,7 +283,7 @@ public static class AttestationEndpoints
BuilderId: doc.GetValue("SupplierId", BsonNull.Value).AsString);
}
private static bool TryResolveTenant(HttpContext context, VexMongoStorageOptions options, out string tenant, out IResult? problem)
private static bool TryResolveTenant(HttpContext context, VexStorageOptions options, out string tenant, out IResult? problem)
{
tenant = options.DefaultTenant;
problem = null;

18
src/Excititor/StellaOps.Excititor.WebService/Endpoints/EvidenceEndpoints.cs
View File

@@ -16,7 +16,7 @@ using MongoDB.Driver;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Canonicalization;
using StellaOps.Excititor.Core.Observations;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Excititor.WebService.Contracts;
using StellaOps.Excititor.WebService.Services;
using StellaOps.Excititor.WebService.Telemetry;
@@ -36,7 +36,7 @@ public static class EvidenceEndpoints
// GET /evidence/vex/list - List evidence exports
app.MapGet("/evidence/vex/list", async (
HttpContext context,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IMongoDatabase database,
TimeProvider timeProvider,
[FromQuery] int? limit,
@@ -114,7 +114,7 @@ public static class EvidenceEndpoints
app.MapGet("/evidence/vex/bundle/{bundleId}", async (
HttpContext context,
string bundleId,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IMongoDatabase database,
TimeProvider timeProvider,
CancellationToken cancellationToken) =>
@@ -191,7 +191,7 @@ public static class EvidenceEndpoints
// GET /evidence/vex/lookup - Lookup evidence for vuln/product pair
app.MapGet("/evidence/vex/lookup", async (
HttpContext context,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IVexObservationProjectionService projectionService,
TimeProvider timeProvider,
[FromQuery] string vulnerabilityId,
@@ -256,7 +256,7 @@ public static class EvidenceEndpoints
app.MapGet("/vuln/evidence/vex/{advisory_key}", async (
HttpContext context,
string advisory_key,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IMongoDatabase database,
TimeProvider timeProvider,
[FromQuery] int? limit,
@@ -446,7 +446,7 @@ public static class EvidenceEndpoints
HttpContext context,
string bundleId,
[FromQuery] string? generation,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
IOptions<AirgapOptions> airgapOptions,
[FromServices] IAirgapImportStore airgapImportStore,
[FromServices] IVexHashingService hashingService,
@@ -528,7 +528,7 @@ public static class EvidenceEndpoints
HttpContext context,
string bundleId,
[FromQuery] string? generation,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
IOptions<AirgapOptions> airgapOptions,
[FromServices] IAirgapImportStore airgapImportStore,
CancellationToken cancellationToken) =>
@@ -575,7 +575,7 @@ public static class EvidenceEndpoints
HttpContext context,
string bundleId,
[FromQuery] string? generation,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
IOptions<AirgapOptions> airgapOptions,
[FromServices] IAirgapImportStore airgapImportStore,
CancellationToken cancellationToken) =>
@@ -679,7 +679,7 @@ public static class EvidenceEndpoints
return (digest, size);
}
private static bool TryResolveTenant(HttpContext context, VexMongoStorageOptions options, out string tenant, out IResult? problem)
private static bool TryResolveTenant(HttpContext context, VexStorageOptions options, out string tenant, out IResult? problem)
{
tenant = options.DefaultTenant;
problem = null;

352
src/Excititor/StellaOps.Excititor.WebService/Endpoints/IngestEndpoints.cs
View File

@@ -20,49 +20,49 @@ internal static class IngestEndpoints
group.MapPost("/reconcile", HandleReconcileAsync);
}
internal static async Task<IResult> HandleInitAsync(
HttpContext httpContext,
ExcititorInitRequest request,
IVexIngestOrchestrator orchestrator,
TimeProvider timeProvider,
CancellationToken cancellationToken)
internal static async Task<IResult> HandleInitAsync(
HttpContext httpContext,
ExcititorInitRequest request,
IVexIngestOrchestrator orchestrator,
TimeProvider timeProvider,
CancellationToken cancellationToken)
{
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
if (scopeResult is not null)
{
return scopeResult;
}
var providerIds = NormalizeProviders(request.Providers);
_ = timeProvider;
var options = new IngestInitOptions(providerIds, request.Resume ?? false);
{
return scopeResult;
}
var providerIds = NormalizeProviders(request.Providers);
_ = timeProvider;
var options = new IngestInitOptions(providerIds, request.Resume ?? false);
var summary = await orchestrator.InitializeAsync(options, cancellationToken).ConfigureAwait(false);
var message = $"Initialized {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed.";
return TypedResults.Ok<object>(new
{
message,
runId = summary.RunId,
startedAt = summary.StartedAt,
completedAt = summary.CompletedAt,
providers = summary.Providers.Select(static provider => new
{
providerId = provider.ProviderId,
displayName = provider.DisplayName,
status = provider.Status,
durationMs = provider.Duration.TotalMilliseconds,
error = provider.Error
})
});
}
return TypedResults.Ok<object>(new
{
message,
runId = summary.RunId,
startedAt = summary.StartedAt,
completedAt = summary.CompletedAt,
providers = summary.Providers.Select(static provider => new
{
providerId = provider.ProviderId,
displayName = provider.DisplayName,
status = provider.Status,
durationMs = provider.Duration.TotalMilliseconds,
error = provider.Error
})
});
}
internal static async Task<IResult> HandleRunAsync(
HttpContext httpContext,
ExcititorIngestRunRequest request,
IVexIngestOrchestrator orchestrator,
TimeProvider timeProvider,
CancellationToken cancellationToken)
internal static async Task<IResult> HandleRunAsync(
HttpContext httpContext,
ExcititorIngestRunRequest request,
IVexIngestOrchestrator orchestrator,
TimeProvider timeProvider,
CancellationToken cancellationToken)
{
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
if (scopeResult is not null)
@@ -72,98 +72,55 @@ internal static class IngestEndpoints
if (!TryParseDateTimeOffset(request.Since, out var since, out var sinceError))
{
return TypedResults.BadRequest<object>(new { message = sinceError });
}
if (!TryParseTimeSpan(request.Window, out var window, out var windowError))
{
return TypedResults.BadRequest<object>(new { message = windowError });
}
_ = timeProvider;
var providerIds = NormalizeProviders(request.Providers);
var options = new IngestRunOptions(
providerIds,
since,
window,
request.Force ?? false);
var summary = await orchestrator.RunAsync(options, cancellationToken).ConfigureAwait(false);
var message = $"Ingest run completed for {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed.";
return TypedResults.Ok<object>(new
{
message,
runId = summary.RunId,
startedAt = summary.StartedAt,
completedAt = summary.CompletedAt,
durationMs = summary.Duration.TotalMilliseconds,
providers = summary.Providers.Select(static provider => new
{
providerId = provider.ProviderId,
status = provider.Status,
documents = provider.Documents,
claims = provider.Claims,
startedAt = provider.StartedAt,
completedAt = provider.CompletedAt,
durationMs = provider.Duration.TotalMilliseconds,
lastDigest = provider.LastDigest,
lastUpdated = provider.LastUpdated,
checkpoint = provider.Checkpoint,
error = provider.Error
})
});
}
return TypedResults.BadRequest<object>(new { message = sinceError });
}
internal static async Task<IResult> HandleResumeAsync(
HttpContext httpContext,
ExcititorIngestResumeRequest request,
IVexIngestOrchestrator orchestrator,
TimeProvider timeProvider,
CancellationToken cancellationToken)
{
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
if (scopeResult is not null)
{
return scopeResult;
}
_ = timeProvider;
var providerIds = NormalizeProviders(request.Providers);
var options = new IngestResumeOptions(providerIds, request.Checkpoint);
var summary = await orchestrator.ResumeAsync(options, cancellationToken).ConfigureAwait(false);
var message = $"Resume run completed for {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed.";
return TypedResults.Ok<object>(new
{
message,
runId = summary.RunId,
startedAt = summary.StartedAt,
completedAt = summary.CompletedAt,
durationMs = summary.Duration.TotalMilliseconds,
providers = summary.Providers.Select(static provider => new
{
providerId = provider.ProviderId,
status = provider.Status,
documents = provider.Documents,
claims = provider.Claims,
startedAt = provider.StartedAt,
completedAt = provider.CompletedAt,
durationMs = provider.Duration.TotalMilliseconds,
since = provider.Since,
checkpoint = provider.Checkpoint,
error = provider.Error
})
});
}
if (!TryParseTimeSpan(request.Window, out var window, out var windowError))
{
return TypedResults.BadRequest<object>(new { message = windowError });
}
internal static async Task<IResult> HandleReconcileAsync(
HttpContext httpContext,
ExcititorReconcileRequest request,
IVexIngestOrchestrator orchestrator,
TimeProvider timeProvider,
CancellationToken cancellationToken)
_ = timeProvider;
var providerIds = NormalizeProviders(request.Providers);
var options = new IngestRunOptions(
providerIds,
since,
window,
request.Force ?? false);
var summary = await orchestrator.RunAsync(options, cancellationToken).ConfigureAwait(false);
var message = $"Ingest run completed for {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed.";
return TypedResults.Ok<object>(new
{
message,
runId = summary.RunId,
startedAt = summary.StartedAt,
completedAt = summary.CompletedAt,
durationMs = summary.Duration.TotalMilliseconds,
providers = summary.Providers.Select(static provider => new
{
providerId = provider.ProviderId,
status = provider.Status,
documents = provider.Documents,
claims = provider.Claims,
startedAt = provider.StartedAt,
completedAt = provider.CompletedAt,
durationMs = provider.Duration.TotalMilliseconds,
lastDigest = provider.LastDigest,
lastUpdated = provider.LastUpdated,
checkpoint = provider.Checkpoint,
error = provider.Error
})
});
}
internal static async Task<IResult> HandleResumeAsync(
HttpContext httpContext,
ExcititorIngestResumeRequest request,
IVexIngestOrchestrator orchestrator,
TimeProvider timeProvider,
CancellationToken cancellationToken)
{
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
if (scopeResult is not null)
@@ -171,40 +128,83 @@ internal static class IngestEndpoints
return scopeResult;
}
if (!TryParseTimeSpan(request.MaxAge, out var maxAge, out var error))
{
return TypedResults.BadRequest<object>(new { message = error });
}
_ = timeProvider;
var providerIds = NormalizeProviders(request.Providers);
var options = new ReconcileOptions(providerIds, maxAge);
var summary = await orchestrator.ReconcileAsync(options, cancellationToken).ConfigureAwait(false);
var message = $"Reconcile completed for {summary.ProviderCount} provider(s); {summary.ReconciledCount} reconciled, {summary.SkippedCount} skipped, {summary.FailureCount} failed.";
return TypedResults.Ok<object>(new
{
message,
runId = summary.RunId,
startedAt = summary.StartedAt,
completedAt = summary.CompletedAt,
durationMs = summary.Duration.TotalMilliseconds,
providers = summary.Providers.Select(static provider => new
{
providerId = provider.ProviderId,
status = provider.Status,
action = provider.Action,
lastUpdated = provider.LastUpdated,
threshold = provider.Threshold,
documents = provider.Documents,
claims = provider.Claims,
error = provider.Error
})
});
}
_ = timeProvider;
var providerIds = NormalizeProviders(request.Providers);
var options = new IngestResumeOptions(providerIds, request.Checkpoint);
internal static ImmutableArray<string> NormalizeProviders(IReadOnlyCollection<string>? providers)
var summary = await orchestrator.ResumeAsync(options, cancellationToken).ConfigureAwait(false);
var message = $"Resume run completed for {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed.";
return TypedResults.Ok<object>(new
{
message,
runId = summary.RunId,
startedAt = summary.StartedAt,
completedAt = summary.CompletedAt,
durationMs = summary.Duration.TotalMilliseconds,
providers = summary.Providers.Select(static provider => new
{
providerId = provider.ProviderId,
status = provider.Status,
documents = provider.Documents,
claims = provider.Claims,
startedAt = provider.StartedAt,
completedAt = provider.CompletedAt,
durationMs = provider.Duration.TotalMilliseconds,
since = provider.Since,
checkpoint = provider.Checkpoint,
error = provider.Error
})
});
}
internal static async Task<IResult> HandleReconcileAsync(
HttpContext httpContext,
ExcititorReconcileRequest request,
IVexIngestOrchestrator orchestrator,
TimeProvider timeProvider,
CancellationToken cancellationToken)
{
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
if (scopeResult is not null)
{
return scopeResult;
}
if (!TryParseTimeSpan(request.MaxAge, out var maxAge, out var error))
{
return TypedResults.BadRequest<object>(new { message = error });
}
_ = timeProvider;
var providerIds = NormalizeProviders(request.Providers);
var options = new ReconcileOptions(providerIds, maxAge);
var summary = await orchestrator.ReconcileAsync(options, cancellationToken).ConfigureAwait(false);
var message = $"Reconcile completed for {summary.ProviderCount} provider(s); {summary.ReconciledCount} reconciled, {summary.SkippedCount} skipped, {summary.FailureCount} failed.";
return TypedResults.Ok<object>(new
{
message,
runId = summary.RunId,
startedAt = summary.StartedAt,
completedAt = summary.CompletedAt,
durationMs = summary.Duration.TotalMilliseconds,
providers = summary.Providers.Select(static provider => new
{
providerId = provider.ProviderId,
status = provider.Status,
action = provider.Action,
lastUpdated = provider.LastUpdated,
threshold = provider.Threshold,
documents = provider.Documents,
claims = provider.Claims,
error = provider.Error
})
});
}
internal static ImmutableArray<string> NormalizeProviders(IReadOnlyCollection<string>? providers)
{
if (providers is null || providers.Count == 0)
{
@@ -225,7 +225,7 @@ internal static class IngestEndpoints
return set.ToImmutableArray();
}
internal static bool TryParseDateTimeOffset(string? value, out DateTimeOffset? result, out string? error)
internal static bool TryParseDateTimeOffset(string? value, out DateTimeOffset? result, out string? error)
{
result = null;
error = null;
@@ -249,7 +249,7 @@ internal static class IngestEndpoints
return false;
}
internal static bool TryParseTimeSpan(string? value, out TimeSpan? result, out string? error)
internal static bool TryParseTimeSpan(string? value, out TimeSpan? result, out string? error)
{
result = null;
error = null;
@@ -269,19 +269,19 @@ internal static class IngestEndpoints
return false;
}
internal sealed record ExcititorInitRequest(IReadOnlyList<string>? Providers, bool? Resume);
internal sealed record ExcititorIngestRunRequest(
IReadOnlyList<string>? Providers,
string? Since,
string? Window,
bool? Force);
internal sealed record ExcititorIngestResumeRequest(
IReadOnlyList<string>? Providers,
string? Checkpoint);
internal sealed record ExcititorReconcileRequest(
IReadOnlyList<string>? Providers,
string? MaxAge);
}
internal sealed record ExcititorInitRequest(IReadOnlyList<string>? Providers, bool? Resume);
internal sealed record ExcititorIngestRunRequest(
IReadOnlyList<string>? Providers,
string? Since,
string? Window,
bool? Force);
internal sealed record ExcititorIngestResumeRequest(
IReadOnlyList<string>? Providers,
string? Checkpoint);
internal sealed record ExcititorReconcileRequest(
IReadOnlyList<string>? Providers,
string? MaxAge);
}

14
src/Excititor/StellaOps.Excititor.WebService/Endpoints/LinksetEndpoints.cs
View File

@@ -10,7 +10,7 @@ using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Core.Canonicalization;
using StellaOps.Excititor.Core.Observations;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Excititor.WebService.Contracts;
using StellaOps.Excititor.WebService.Services;
using StellaOps.Excititor.WebService.Telemetry;
@@ -32,7 +32,7 @@ public static class LinksetEndpoints
// GET /vex/linksets - List linksets with filters
group.MapGet("", async (
HttpContext context,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IVexLinksetStore linksetStore,
[FromQuery] int? limit,
[FromQuery] string? cursor,
@@ -124,7 +124,7 @@ public static class LinksetEndpoints
group.MapGet("/{linksetId}", async (
HttpContext context,
string linksetId,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IVexLinksetStore linksetStore,
CancellationToken cancellationToken) =>
{
@@ -166,7 +166,7 @@ public static class LinksetEndpoints
// GET /vex/linksets/lookup - Lookup linkset by vulnerability and product
group.MapGet("/lookup", async (
HttpContext context,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IVexLinksetStore linksetStore,
[FromQuery] string? vulnerabilityId,
[FromQuery] string? productKey,
@@ -211,7 +211,7 @@ public static class LinksetEndpoints
// GET /vex/linksets/count - Get linkset counts for tenant
group.MapGet("/count", async (
HttpContext context,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IVexLinksetStore linksetStore,
CancellationToken cancellationToken) =>
{
@@ -240,7 +240,7 @@ public static class LinksetEndpoints
// GET /vex/linksets/conflicts - List linksets with conflicts (shorthand)
group.MapGet("/conflicts", async (
HttpContext context,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IVexLinksetStore linksetStore,
[FromQuery] int? limit,
CancellationToken cancellationToken) =>
@@ -317,7 +317,7 @@ public static class LinksetEndpoints
private static bool TryResolveTenant(
HttpContext context,
VexMongoStorageOptions options,
VexStorageOptions options,
out string tenant,
out IResult? problem)
{

110
src/Excititor/StellaOps.Excititor.WebService/Endpoints/MirrorEndpoints.cs
View File

@@ -8,8 +8,8 @@ using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Export;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.WebService.Services;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Excititor.WebService.Services;
namespace StellaOps.Excititor.WebService.Endpoints;
@@ -98,13 +98,13 @@ internal static class MirrorEndpoints
}
var resolvedExports = new List<MirrorExportIndexEntry>();
foreach (var exportOption in domain.Exports)
{
if (!MirrorExportPlanner.TryBuild(exportOption, out var plan, out var error))
{
resolvedExports.Add(new MirrorExportIndexEntry(
exportOption.Key,
null,
foreach (var exportOption in domain.Exports)
{
if (!MirrorExportPlanner.TryBuild(exportOption, out var plan, out var error))
{
resolvedExports.Add(new MirrorExportIndexEntry(
exportOption.Key,
null,
null,
exportOption.Format,
null,
@@ -116,7 +116,7 @@ internal static class MirrorEndpoints
continue;
}
var manifest = await exportStore.FindAsync(plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false);
var manifest = await exportStore.FindAsync(plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false);
if (manifest is null)
{
@@ -177,16 +177,16 @@ internal static class MirrorEndpoints
return Results.Unauthorized();
}
if (!TryFindExport(domain, exportKey, out var exportOptions))
{
return Results.NotFound();
}
if (!MirrorExportPlanner.TryBuild(exportOptions, out var plan, out var error))
{
await WritePlainTextAsync(httpContext, error ?? "invalid_export_configuration", StatusCodes.Status503ServiceUnavailable, cancellationToken).ConfigureAwait(false);
return Results.Empty;
}
if (!TryFindExport(domain, exportKey, out var exportOptions))
{
return Results.NotFound();
}
if (!MirrorExportPlanner.TryBuild(exportOptions, out var plan, out var error))
{
await WritePlainTextAsync(httpContext, error ?? "invalid_export_configuration", StatusCodes.Status503ServiceUnavailable, cancellationToken).ConfigureAwait(false);
return Results.Empty;
}
var manifest = await exportStore.FindAsync(plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false);
if (manifest is null)
@@ -241,10 +241,10 @@ internal static class MirrorEndpoints
return Results.Empty;
}
if (!TryFindExport(domain, exportKey, out var exportOptions) || !MirrorExportPlanner.TryBuild(exportOptions, out var plan, out _))
{
return Results.NotFound();
}
if (!TryFindExport(domain, exportKey, out var exportOptions) || !MirrorExportPlanner.TryBuild(exportOptions, out var plan, out _))
{
return Results.NotFound();
}
var manifest = await exportStore.FindAsync(plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false);
if (manifest is null)
@@ -286,36 +286,36 @@ internal static class MirrorEndpoints
return domain is not null;
}
private static bool TryFindExport(MirrorDomainOptions domain, string exportKey, out MirrorExportOptions export)
{
export = domain.Exports.FirstOrDefault(e => string.Equals(e.Key, exportKey, StringComparison.OrdinalIgnoreCase))!;
return export is not null;
}
private static bool TryFindExport(MirrorDomainOptions domain, string exportKey, out MirrorExportOptions export)
{
export = domain.Exports.FirstOrDefault(e => string.Equals(e.Key, exportKey, StringComparison.OrdinalIgnoreCase))!;
return export is not null;
}
private static string ResolveContentType(VexExportFormat format)
=> format switch
{
VexExportFormat.Json => "application/json",
VexExportFormat.JsonLines => "application/jsonl",
VexExportFormat.OpenVex => "application/json",
VexExportFormat.Csaf => "application/json",
VexExportFormat.CycloneDx => "application/json",
_ => "application/octet-stream",
};
private static string ResolveContentType(VexExportFormat format)
=> format switch
{
VexExportFormat.Json => "application/json",
VexExportFormat.JsonLines => "application/jsonl",
VexExportFormat.OpenVex => "application/json",
VexExportFormat.Csaf => "application/json",
VexExportFormat.CycloneDx => "application/json",
_ => "application/octet-stream",
};
private static string BuildDownloadFileName(string domainId, string exportKey, VexExportFormat format)
{
var builder = new StringBuilder(domainId.Length + exportKey.Length + 8);
builder.Append(domainId).Append('-').Append(exportKey);
builder.Append(format switch
{
VexExportFormat.Json => ".json",
VexExportFormat.JsonLines => ".jsonl",
VexExportFormat.OpenVex => ".openvex.json",
VexExportFormat.Csaf => ".csaf.json",
VexExportFormat.CycloneDx => ".cyclonedx.json",
_ => ".bin",
});
builder.Append(format switch
{
VexExportFormat.Json => ".json",
VexExportFormat.JsonLines => ".jsonl",
VexExportFormat.OpenVex => ".openvex.json",
VexExportFormat.Csaf => ".csaf.json",
VexExportFormat.CycloneDx => ".cyclonedx.json",
_ => ".bin",
});
return builder.ToString();
}
@@ -326,15 +326,15 @@ internal static class MirrorEndpoints
await context.Response.WriteAsync(message, cancellationToken);
}
private static async Task WriteJsonAsync<T>(HttpContext context, T payload, int statusCode, CancellationToken cancellationToken)
{
context.Response.StatusCode = statusCode;
context.Response.ContentType = "application/json";
var json = VexCanonicalJsonSerializer.Serialize(payload);
await context.Response.WriteAsync(json, cancellationToken);
private static async Task WriteJsonAsync<T>(HttpContext context, T payload, int statusCode, CancellationToken cancellationToken)
{
context.Response.StatusCode = statusCode;
context.Response.ContentType = "application/json";
var json = VexCanonicalJsonSerializer.Serialize(payload);
await context.Response.WriteAsync(json, cancellationToken);
}
}
}
internal sealed record MirrorDomainListResponse(IReadOnlyList<MirrorDomainSummary> Domains);

2
src/Excititor/StellaOps.Excititor.WebService/Endpoints/MirrorRegistrationEndpoints.cs
View File

@@ -8,7 +8,7 @@ using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Excititor.WebService.Contracts;
namespace StellaOps.Excititor.WebService.Endpoints;

10
src/Excititor/StellaOps.Excititor.WebService/Endpoints/ObservationEndpoints.cs
View File

@@ -6,7 +6,7 @@ using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Core.Observations;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Excititor.WebService.Contracts;
using StellaOps.Excititor.WebService.Services;
@@ -26,7 +26,7 @@ public static class ObservationEndpoints
// GET /vex/observations - List observations with filters
group.MapGet("", async (
HttpContext context,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IVexObservationStore observationStore,
TimeProvider timeProvider,
[FromQuery] int? limit,
@@ -98,7 +98,7 @@ public static class ObservationEndpoints
group.MapGet("/{observationId}", async (
HttpContext context,
string observationId,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IVexObservationStore observationStore,
CancellationToken cancellationToken) =>
{
@@ -140,7 +140,7 @@ public static class ObservationEndpoints
// GET /vex/observations/count - Get observation count for tenant
group.MapGet("/count", async (
HttpContext context,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IVexObservationStore observationStore,
CancellationToken cancellationToken) =>
{
@@ -230,7 +230,7 @@ public static class ObservationEndpoints
private static bool TryResolveTenant(
HttpContext context,
VexMongoStorageOptions options,
VexStorageOptions options,
out string tenant,
out IResult? problem)
{

6
src/Excititor/StellaOps.Excititor.WebService/Endpoints/PolicyEndpoints.cs
View File

@@ -11,7 +11,7 @@ using Microsoft.Extensions.Options;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Canonicalization;
using StellaOps.Excititor.Core.Orchestration;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Excititor.WebService.Contracts;
using StellaOps.Excititor.WebService.Services;
@@ -33,7 +33,7 @@ public static class PolicyEndpoints
private static async Task<IResult> LookupVexAsync(
HttpContext context,
[FromBody] PolicyVexLookupRequest request,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IVexClaimStore claimStore,
TimeProvider timeProvider,
CancellationToken cancellationToken)
@@ -174,7 +174,7 @@ public static class PolicyEndpoints
private static bool TryResolveTenant(
HttpContext context,
VexMongoStorageOptions options,
VexStorageOptions options,
out string tenant,
out IResult? problem)
{

86
src/Excititor/StellaOps.Excititor.WebService/Endpoints/ResolveEndpoint.cs
View File

@@ -5,55 +5,55 @@ using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Excititor.Attestation;
using StellaOps.Excititor.Attestation.Dsse;
using StellaOps.Excititor.Attestation.Signing;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Policy;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.WebService.Services;
internal static class ResolveEndpoint
{
private const int MaxSubjectPairs = 256;
private const string ReadScope = "vex.read";
public static void MapResolveEndpoint(WebApplication app)
{
app.MapPost("/excititor/resolve", HandleResolveAsync);
}
using System.Text;
using System.Text.Json;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Excititor.Attestation;
using StellaOps.Excititor.Attestation.Dsse;
using StellaOps.Excititor.Attestation.Signing;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Policy;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Excititor.WebService.Services;
internal static class ResolveEndpoint
{
private const int MaxSubjectPairs = 256;
private const string ReadScope = "vex.read";
public static void MapResolveEndpoint(WebApplication app)
{
app.MapPost("/excititor/resolve", HandleResolveAsync);
}
private static async Task<IResult> HandleResolveAsync(
VexResolveRequest request,
HttpContext httpContext,
IVexClaimStore claimStore,
IVexConsensusStore consensusStore,
IVexProviderStore providerStore,
IVexPolicyProvider policyProvider,
TimeProvider timeProvider,
ILoggerFactory loggerFactory,
IVexAttestationClient? attestationClient,
CancellationToken cancellationToken)
{
var scopeResult = ScopeAuthorization.RequireScope(httpContext, ReadScope);
if (scopeResult is not null)
{
return scopeResult;
}
if (request is null)
{
return Results.BadRequest("Request payload is required.");
}
var logger = loggerFactory.CreateLogger("ResolveEndpoint");
var signer = httpContext.RequestServices.GetService<IVexSigner>();
IVexProviderStore providerStore,
IVexPolicyProvider policyProvider,
TimeProvider timeProvider,
ILoggerFactory loggerFactory,
IVexAttestationClient? attestationClient,
CancellationToken cancellationToken)
{
var scopeResult = ScopeAuthorization.RequireScope(httpContext, ReadScope);
if (scopeResult is not null)
{
return scopeResult;
}
if (request is null)
{
return Results.BadRequest("Request payload is required.");
}
var logger = loggerFactory.CreateLogger("ResolveEndpoint");
var signer = httpContext.RequestServices.GetService<IVexSigner>();
var productKeys = NormalizeValues(request.ProductKeys, request.Purls);
var vulnerabilityIds = NormalizeValues(request.VulnerabilityIds);

12
src/Excititor/StellaOps.Excititor.WebService/Endpoints/RiskFeedEndpoints.cs
View File

@@ -6,7 +6,7 @@ using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.RiskFeed;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Excititor.WebService.Services;
namespace StellaOps.Excititor.WebService.Endpoints;
@@ -25,7 +25,7 @@ public static class RiskFeedEndpoints
// POST /risk/v1/feed - Generate risk feed
group.MapPost("/feed", async (
HttpContext context,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IRiskFeedService riskFeedService,
[FromBody] RiskFeedRequestDto request,
CancellationToken cancellationToken) =>
@@ -67,7 +67,7 @@ public static class RiskFeedEndpoints
// GET /risk/v1/feed/item - Get single risk feed item
group.MapGet("/feed/item", async (
HttpContext context,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IRiskFeedService riskFeedService,
[FromQuery] string? advisoryKey,
[FromQuery] string? artifact,
@@ -112,7 +112,7 @@ public static class RiskFeedEndpoints
group.MapGet("/feed/by-advisory/{advisoryKey}", async (
HttpContext context,
string advisoryKey,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IRiskFeedService riskFeedService,
[FromQuery] int? limit,
CancellationToken cancellationToken) =>
@@ -153,7 +153,7 @@ public static class RiskFeedEndpoints
group.MapGet("/feed/by-artifact/{**artifact}", async (
HttpContext context,
string artifact,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IRiskFeedService riskFeedService,
[FromQuery] int? limit,
CancellationToken cancellationToken) =>
@@ -235,7 +235,7 @@ public static class RiskFeedEndpoints
private static bool TryResolveTenant(
HttpContext context,
VexMongoStorageOptions options,
VexStorageOptions options,
out string tenant,
out IResult? problem)
{

71
src/Excititor/StellaOps.Excititor.WebService/Extensions/VexRawDocumentMapper.cs Normal file
View File

@@ -0,0 +1,71 @@
using System.Collections.Immutable;
using System.Text.Json;
using StellaOps.Concelier.RawModels;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Storage;
namespace StellaOps.Excititor.WebService.Extensions;
internal static class VexRawDocumentMapper
{
public static VexRawDocument ToRawModel(VexRawRecord record, string defaultTenant)
{
ArgumentNullException.ThrowIfNull(record);
var metadata = record.Metadata ?? ImmutableDictionary<string, string>.Empty;
var tenant = Get(metadata, "tenant", record.Tenant) ?? defaultTenant;
var source = new RawSourceMetadata(
Vendor: Get(metadata, "source.vendor", record.ProviderId) ?? record.ProviderId,
Connector: Get(metadata, "source.connector", record.ProviderId) ?? record.ProviderId,
ConnectorVersion: Get(metadata, "source.connector_version", "unknown") ?? "unknown",
Stream: Get(metadata, "source.stream", record.Format.ToString().ToLowerInvariant()));
var signature = new RawSignatureMetadata(
Present: string.Equals(Get(metadata, "signature.present"), "true", StringComparison.OrdinalIgnoreCase),
Format: Get(metadata, "signature.format"),
KeyId: Get(metadata, "signature.key_id"),
Signature: Get(metadata, "signature.sig"),
Certificate: Get(metadata, "signature.certificate"),
Digest: Get(metadata, "signature.digest"));
var upstream = new RawUpstreamMetadata(
UpstreamId: Get(metadata, "upstream.id", record.Digest) ?? record.Digest,
DocumentVersion: Get(metadata, "upstream.version"),
RetrievedAt: record.RetrievedAt,
ContentHash: Get(metadata, "upstream.content_hash", record.Digest) ?? record.Digest,
Signature: signature,
Provenance: metadata);
var content = new RawContent(
Format: record.Format.ToString().ToLowerInvariant(),
SpecVersion: Get(metadata, "content.spec_version"),
Raw: ParseJson(record.Content),
Encoding: Get(metadata, "content.encoding"));
return new VexRawDocument(
tenant,
source,
upstream,
content,
new RawLinkset(),
statements: null,
supersedes: record.SupersedesDigest);
}
private static string? Get(IReadOnlyDictionary<string, string> metadata, string key, string? fallback = null)
{
if (metadata.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
{
return value;
}
return fallback;
}
private static JsonElement ParseJson(ReadOnlyMemory<byte> content)
{
using var document = JsonDocument.Parse(content);
return document.RootElement.Clone();
}
}

26
src/Excititor/StellaOps.Excititor.WebService/Program.Helpers.cs
View File

@@ -6,17 +6,16 @@ using System.Linq;
using System.Text;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Primitives;
using MongoDB.Bson;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Aoc;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Excititor.WebService.Contracts;
using StellaOps.Excititor.WebService.Services;
public partial class Program
{
private const string TenantHeaderName = "X-Stella-Tenant";
private static bool TryResolveTenant(HttpContext context, VexMongoStorageOptions options, bool requireHeader, out string tenant, out IResult? problem)
private static bool TryResolveTenant(HttpContext context, VexStorageOptions options, bool requireHeader, out string tenant, out IResult? problem)
{
tenant = options.DefaultTenant;
problem = null;
@@ -51,27 +50,6 @@ public partial class Program
return true;
}
private static IReadOnlyDictionary<string, string> ReadMetadata(BsonValue value)
{
if (value is not BsonDocument doc || doc.ElementCount == 0)
{
return new Dictionary<string, string>(StringComparer.Ordinal);
}
var result = new Dictionary<string, string>(StringComparer.Ordinal);
foreach (var element in doc.Elements)
{
if (string.IsNullOrWhiteSpace(element.Name))
{
continue;
}
result[element.Name] = element.Value?.ToString() ?? string.Empty;
}
return result;
}
private static bool TryDecodeCursor(string? cursor, out DateTimeOffset timestamp, out string digest)
{
timestamp = default;

222
src/Excititor/StellaOps.Excititor.WebService/Program.cs
View File

@@ -27,28 +27,27 @@ using StellaOps.Excititor.Formats.CSAF;
using StellaOps.Excititor.Formats.CycloneDX;
using StellaOps.Excititor.Formats.OpenVEX;
using StellaOps.Excititor.Policy;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.Storage.Postgres;
using StellaOps.Excititor.WebService.Endpoints;
using StellaOps.Excititor.WebService.Extensions;
using StellaOps.Excititor.WebService.Options;
using StellaOps.Excititor.WebService.Services;
using StellaOps.Excititor.Core.Aoc;
using StellaOps.Excititor.WebService.Telemetry;
using MongoDB.Driver;
using MongoDB.Bson;
using Microsoft.Extensions.Caching.Memory;
using StellaOps.Excititor.WebService.Contracts;
using System.Globalization;
using StellaOps.Excititor.WebService.Graph;
using StellaOps.Excititor.Core.Storage;
var builder = WebApplication.CreateBuilder(args);
var configuration = builder.Configuration;
var services = builder.Services;
services.AddOptions<VexMongoStorageOptions>()
.Bind(configuration.GetSection("Excititor:Storage:Mongo"))
services.AddOptions<VexStorageOptions>()
.Bind(configuration.GetSection("Excititor:Storage"))
.ValidateOnStart();
services.AddExcititorMongoStorage();
services.AddExcititorPostgresStorage(configuration);
services.AddCsafNormalizer();
services.AddCycloneDxNormalizer();
services.AddOpenVexNormalizer();
@@ -147,7 +146,7 @@ app.UseObservabilityHeaders();
app.MapGet("/excititor/status", async (HttpContext context,
IEnumerable<IVexArtifactStore> artifactStores,
IOptions<VexMongoStorageOptions> mongoOptions,
IOptions<VexStorageOptions> mongoOptions,
TimeProvider timeProvider) =>
{
var payload = new StatusResponse(
@@ -1260,7 +1259,7 @@ app.MapPost("/excititor/admin/backfill-statements", async (
app.MapGet("/console/vex", async (
HttpContext context,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
IVexObservationQueryService queryService,
ConsoleTelemetry telemetry,
IMemoryCache cache,
@@ -1459,7 +1458,7 @@ var response = new GraphLinkoutsResponse(items, notFound);
app.MapGet("/v1/graph/status", async (
HttpContext context,
[FromQuery(Name = "purl")] string[]? purls,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
IOptions<GraphOptions> graphOptions,
IVexObservationQueryService queryService,
IMemoryCache cache,
@@ -1519,7 +1518,7 @@ app.MapGet("/v1/graph/overlays", async (
HttpContext context,
[FromQuery(Name = "purl")] string[]? purls,
[FromQuery] bool includeJustifications,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
IOptions<GraphOptions> graphOptions,
IVexObservationQueryService queryService,
IMemoryCache cache,
@@ -1580,7 +1579,7 @@ app.MapGet("/v1/graph/observations", async (
[FromQuery] bool includeJustifications,
[FromQuery] int? limitPerPurl,
[FromQuery] string? cursor,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
IOptions<GraphOptions> graphOptions,
IVexObservationQueryService queryService,
CancellationToken cancellationToken) =>
@@ -1638,7 +1637,7 @@ app.MapPost("/ingest/vex", async (
HttpContext context,
VexIngestRequest request,
IVexRawStore rawStore,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
TimeProvider timeProvider,
ILogger<Program> logger,
CancellationToken cancellationToken) =>
@@ -1692,8 +1691,8 @@ app.MapPost("/ingest/vex", async (
app.MapGet("/vex/raw", async (
HttpContext context,
IMongoDatabase database,
IOptions<VexMongoStorageOptions> storageOptions,
IVexRawStore rawStore,
IOptions<VexStorageOptions> storageOptions,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
@@ -1702,132 +1701,69 @@ app.MapGet("/vex/raw", async (
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out _, out var tenantError))
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out var tenant, out var tenantError))
{
return tenantError;
}
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
var query = context.Request.Query;
var filters = new List<FilterDefinition<BsonDocument>>();
var builder = Builders<BsonDocument>.Filter;
var providerFilter = BuildStringFilterSet(query["providerId"]);
var digestFilter = BuildStringFilterSet(query["digest"]);
var formatFilter = query.TryGetValue("format", out var formats)
? formats
.Where(static f => !string.IsNullOrWhiteSpace(f))
.Select(static f => Enum.TryParse<VexDocumentFormat>(f, true, out var parsed) ? parsed : VexDocumentFormat.Unknown)
.Where(static f => f != VexDocumentFormat.Unknown)
.ToArray()
: Array.Empty<VexDocumentFormat>();
if (query.TryGetValue("providerId", out var providerValues))
{
var providers = providerValues
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Select(static value => value!.Trim())
.ToArray();
if (providers.Length > 0)
{
filters.Add(builder.In("ProviderId", providers));
}
}
if (query.TryGetValue("digest", out var digestValues))
{
var digests = digestValues
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Select(static value => value!.Trim())
.ToArray();
if (digests.Length > 0)
{
filters.Add(builder.In("Digest", digests));
}
}
if (query.TryGetValue("format", out var formatValues))
{
var formats = formatValues
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Select(static value => value!.Trim().ToLowerInvariant())
.ToArray();
if (formats.Length > 0)
{
filters.Add(builder.In("Format", formats));
}
}
if (query.TryGetValue("since", out var sinceValues) && DateTimeOffset.TryParse(sinceValues.FirstOrDefault(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var sinceValue))
{
filters.Add(builder.Gte("RetrievedAt", sinceValue.UtcDateTime));
}
var since = ParseSinceTimestamp(query["since"]);
var cursorToken = query.TryGetValue("cursor", out var cursorValues) ? cursorValues.FirstOrDefault() : null;
DateTime? cursorTimestamp = null;
string? cursorDigest = null;
if (!string.IsNullOrWhiteSpace(cursorToken) && TryDecodeCursor(cursorToken, out var cursorTime, out var cursorId))
VexRawCursor? cursor = null;
if (!string.IsNullOrWhiteSpace(cursorToken) &&
TryDecodeCursor(cursorToken, out var cursorTime, out var cursorId))
{
cursorTimestamp = cursorTime.UtcDateTime;
cursorDigest = cursorId;
cursor = new VexRawCursor(cursorTime, cursorId);
}
if (cursorTimestamp is not null && cursorDigest is not null)
{
var ltTime = builder.Lt("RetrievedAt", cursorTimestamp.Value);
var eqTimeLtDigest = builder.And(
builder.Eq("RetrievedAt", cursorTimestamp.Value),
builder.Lt("Digest", cursorDigest));
filters.Add(builder.Or(ltTime, eqTimeLtDigest));
}
var limit = ResolveLimit(query["limit"], defaultValue: 50, min: 1, max: 200);
var limit = 50;
if (query.TryGetValue("limit", out var limitValues) && int.TryParse(limitValues.FirstOrDefault(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var requestedLimit))
{
limit = Math.Clamp(requestedLimit, 1, 200);
}
var page = await rawStore.QueryAsync(
new VexRawQuery(
tenant,
providerFilter,
digestFilter,
formatFilter,
since,
Until: null,
cursor,
limit),
cancellationToken).ConfigureAwait(false);
var filter = filters.Count == 0 ? builder.Empty : builder.And(filters);
var sort = Builders<BsonDocument>.Sort.Descending("RetrievedAt").Descending("Digest");
var documents = await collection
.Find(filter)
.Sort(sort)
.Limit(limit)
.Project(Builders<BsonDocument>.Projection.Include("Digest").Include("ProviderId").Include("Format").Include("SourceUri").Include("RetrievedAt").Include("Metadata").Include("GridFsObjectId"))
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var summaries = page.Items
.Select(summary => new VexRawSummaryResponse(
summary.Digest,
summary.ProviderId,
summary.Format.ToString().ToLowerInvariant(),
summary.SourceUri.ToString(),
summary.RetrievedAt,
summary.InlineContent,
summary.Metadata))
.ToList();
var summaries = new List<VexRawSummaryResponse>(documents.Count);
foreach (var document in documents)
{
var digest = document.TryGetValue("Digest", out var digestValue) && digestValue.IsString ? digestValue.AsString : string.Empty;
var providerId = document.TryGetValue("ProviderId", out var providerValue) && providerValue.IsString ? providerValue.AsString : string.Empty;
var format = document.TryGetValue("Format", out var formatValue) && formatValue.IsString ? formatValue.AsString : string.Empty;
var sourceUri = document.TryGetValue("SourceUri", out var sourceValue) && sourceValue.IsString ? sourceValue.AsString : string.Empty;
var retrievedAt = document.TryGetValue("RetrievedAt", out var retrievedValue) && retrievedValue is BsonDateTime bsonDate
? bsonDate.ToUniversalTime()
: DateTime.UtcNow;
var metadata = ReadMetadata(document.TryGetValue("Metadata", out var metadataValue) ? metadataValue : BsonNull.Value);
var inlineContent = !document.TryGetValue("GridFsObjectId", out var gridId) || gridId.IsBsonNull || (gridId.IsString && string.IsNullOrWhiteSpace(gridId.AsString));
var nextCursor = page.NextCursor is null
? null
: EncodeCursor(page.NextCursor.RetrievedAt.UtcDateTime, page.NextCursor.Digest);
summaries.Add(new VexRawSummaryResponse(
digest,
providerId,
format,
sourceUri,
new DateTimeOffset(retrievedAt),
inlineContent,
metadata));
}
var hasMore = documents.Count == limit;
string? nextCursor = null;
if (hasMore && documents.Count > 0)
{
var last = documents[^1];
var lastTime = last.GetValue("RetrievedAt", BsonNull.Value).ToUniversalTime();
var lastDigest = last.GetValue("Digest", BsonNull.Value).AsString;
nextCursor = EncodeCursor(lastTime, lastDigest);
}
return Results.Json(new VexRawListResponse(summaries, nextCursor, hasMore));
return Results.Json(new VexRawListResponse(summaries, nextCursor, page.HasMore));
});
app.MapGet("/vex/raw/{digest}", async (
string digest,
HttpContext context,
IVexRawStore rawStore,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
@@ -1861,7 +1797,7 @@ app.MapGet("/vex/raw/{digest}/provenance", async (
string digest,
HttpContext context,
IVexRawStore rawStore,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
@@ -1901,7 +1837,7 @@ app.MapGet("/v1/vex/observations/{vulnerabilityId}/{productKey}", async (
string vulnerabilityId,
string productKey,
[FromServices] IVexObservationProjectionService projectionService,
[FromServices] IOptions<VexMongoStorageOptions> storageOptions,
[FromServices] IOptions<VexStorageOptions> storageOptions,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
@@ -1977,7 +1913,7 @@ app.MapGet("/v1/vex/observations/{vulnerabilityId}/{productKey}", async (
app.MapGet("/v1/vex/evidence/chunks", async (
HttpContext context,
[FromServices] IVexEvidenceChunkService chunkService,
[FromServices] IOptions<VexMongoStorageOptions> storageOptions,
[FromServices] IOptions<VexStorageOptions> storageOptions,
[FromServices] ChunkTelemetry chunkTelemetry,
[FromServices] ILogger<VexEvidenceChunkRequest> logger,
[FromServices] TimeProvider timeProvider,
@@ -2083,10 +2019,9 @@ app.MapGet("/v1/vex/evidence/chunks", async (
app.MapPost("/aoc/verify", async (
HttpContext context,
VexAocVerifyRequest? request,
IMongoDatabase database,
IVexRawStore rawStore,
IVexRawWriteGuard guard,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
TimeProvider timeProvider,
CancellationToken cancellationToken) =>
{
@@ -2119,33 +2054,26 @@ app.MapPost("/aoc/verify", async (
.Select(static value => value!.Trim())
.ToArray();
var builder = Builders<BsonDocument>.Filter;
var filter = builder.And(
builder.Gte("RetrievedAt", since),
builder.Lte("RetrievedAt", until));
if (sources is { Length: > 0 })
{
filter &= builder.In("ProviderId", sources);
}
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
var digests = await collection
.Find(filter)
.Sort(Builders<BsonDocument>.Sort.Descending("RetrievedAt"))
.Limit(limit)
.Project(Builders<BsonDocument>.Projection.Include("Digest").Include("RetrievedAt").Include("ProviderId"))
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var page = await rawStore.QueryAsync(
new VexRawQuery(
tenant,
sources ?? Array.Empty<string>(),
Array.Empty<string>(),
Array.Empty<VexDocumentFormat>(),
since: new DateTimeOffset(since, TimeSpan.Zero),
until: new DateTimeOffset(until, TimeSpan.Zero),
cursor: null,
limit),
cancellationToken).ConfigureAwait(false);
var checkedCount = 0;
var violationMap = new Dictionary<string, (int Count, List<VexAocVerifyViolationExample> Examples)>(StringComparer.OrdinalIgnoreCase);
const int MaxExamplesPerCode = 5;
foreach (var digestDocument in digests)
foreach (var item in page.Items)
{
var digestValue = digestDocument.GetValue("Digest", BsonNull.Value).AsString;
var provider = digestDocument.GetValue("ProviderId", BsonNull.Value).AsString;
var digestValue = item.Digest;
var provider = item.ProviderId;
var domainDocument = await rawStore.FindByDigestAsync(digestValue, cancellationToken).ConfigureAwait(false);
if (domainDocument is null)
@@ -2202,7 +2130,7 @@ app.MapPost("/aoc/verify", async (
new VexAocVerifyChecked(0, checkedCount),
violations,
new VexAocVerifyMetrics(checkedCount, violations.Sum(v => v.Count)),
digests.Count == limit);
page.HasMore);
return Results.Json(response);
});
@@ -2225,7 +2153,7 @@ app.MapGet("/obs/excititor/health", async (
// VEX timeline SSE (WEB-OBS-52-001)
app.MapGet("/obs/excititor/timeline", async (
HttpContext context,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IVexTimelineEventStore timelineStore,
TimeProvider timeProvider,
ILoggerFactory loggerFactory,

4
src/Excititor/StellaOps.Excititor.WebService/Properties/AssemblyInfo.cs
View File

@@ -1,4 +1,4 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Excititor.WebService.Tests")]
[assembly: InternalsVisibleTo("StellaOps.Excititor.Core.UnitTests")]
[assembly: InternalsVisibleTo("StellaOps.Excititor.WebService.Tests")]
[assembly: InternalsVisibleTo("StellaOps.Excititor.Core.UnitTests")]

2
src/Excititor/StellaOps.Excititor.WebService/Services/ExcititorHealthService.cs
View File

@@ -7,7 +7,7 @@ using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Excititor.WebService.Options;
namespace StellaOps.Excititor.WebService.Services;

2
src/Excititor/StellaOps.Excititor.WebService/Services/VexEvidenceChunkService.cs
View File

@@ -6,7 +6,7 @@ using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Excititor.WebService.Contracts;
namespace StellaOps.Excititor.WebService.Services;

253
src/Excititor/StellaOps.Excititor.WebService/Services/VexIngestOrchestrator.cs
View File

@@ -1,14 +1,14 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.Core.Storage;
namespace StellaOps.Excititor.WebService.Services;
@@ -23,50 +23,47 @@ internal interface IVexIngestOrchestrator
Task<ReconcileSummary> ReconcileAsync(ReconcileOptions options, CancellationToken cancellationToken);
}
internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
{
private readonly IServiceProvider _serviceProvider;
private readonly IReadOnlyDictionary<string, IVexConnector> _connectors;
private readonly IVexRawStore _rawStore;
private readonly IVexClaimStore _claimStore;
private readonly IVexProviderStore _providerStore;
private readonly IVexConnectorStateRepository _stateRepository;
private readonly IVexNormalizerRouter _normalizerRouter;
private readonly IVexSignatureVerifier _signatureVerifier;
private readonly IVexMongoSessionProvider _sessionProvider;
private readonly TimeProvider _timeProvider;
private readonly ILogger<VexIngestOrchestrator> _logger;
private readonly string _defaultTenant;
internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
{
private readonly IServiceProvider _serviceProvider;
private readonly IReadOnlyDictionary<string, IVexConnector> _connectors;
private readonly IVexRawStore _rawStore;
private readonly IVexClaimStore _claimStore;
private readonly IVexProviderStore _providerStore;
private readonly IVexConnectorStateRepository _stateRepository;
private readonly IVexNormalizerRouter _normalizerRouter;
private readonly IVexSignatureVerifier _signatureVerifier;
private readonly TimeProvider _timeProvider;
private readonly ILogger<VexIngestOrchestrator> _logger;
private readonly string _defaultTenant;
public VexIngestOrchestrator(
IServiceProvider serviceProvider,
IEnumerable<IVexConnector> connectors,
IVexRawStore rawStore,
IVexClaimStore claimStore,
IVexProviderStore providerStore,
IVexConnectorStateRepository stateRepository,
IVexNormalizerRouter normalizerRouter,
IVexSignatureVerifier signatureVerifier,
IVexMongoSessionProvider sessionProvider,
TimeProvider timeProvider,
IOptions<VexMongoStorageOptions> storageOptions,
ILogger<VexIngestOrchestrator> logger)
{
_serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
_rawStore = rawStore ?? throw new ArgumentNullException(nameof(rawStore));
_claimStore = claimStore ?? throw new ArgumentNullException(nameof(claimStore));
public VexIngestOrchestrator(
IServiceProvider serviceProvider,
IEnumerable<IVexConnector> connectors,
IVexRawStore rawStore,
IVexClaimStore claimStore,
IVexProviderStore providerStore,
IVexConnectorStateRepository stateRepository,
IVexNormalizerRouter normalizerRouter,
IVexSignatureVerifier signatureVerifier,
TimeProvider timeProvider,
IOptions<VexStorageOptions> storageOptions,
ILogger<VexIngestOrchestrator> logger)
{
_serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
_rawStore = rawStore ?? throw new ArgumentNullException(nameof(rawStore));
_claimStore = claimStore ?? throw new ArgumentNullException(nameof(claimStore));
_providerStore = providerStore ?? throw new ArgumentNullException(nameof(providerStore));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_normalizerRouter = normalizerRouter ?? throw new ArgumentNullException(nameof(normalizerRouter));
_signatureVerifier = signatureVerifier ?? throw new ArgumentNullException(nameof(signatureVerifier));
_sessionProvider = sessionProvider ?? throw new ArgumentNullException(nameof(sessionProvider));
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
var optionsValue = (storageOptions ?? throw new ArgumentNullException(nameof(storageOptions))).Value
?? throw new ArgumentNullException(nameof(storageOptions));
_defaultTenant = string.IsNullOrWhiteSpace(optionsValue.DefaultTenant)
? "default"
: optionsValue.DefaultTenant.Trim();
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
var optionsValue = (storageOptions ?? throw new ArgumentNullException(nameof(storageOptions))).Value
?? throw new ArgumentNullException(nameof(storageOptions));
_defaultTenant = string.IsNullOrWhiteSpace(optionsValue.DefaultTenant)
? "default"
: optionsValue.DefaultTenant.Trim();
if (connectors is null)
{
@@ -86,8 +83,6 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
var startedAt = _timeProvider.GetUtcNow();
var results = ImmutableArray.CreateBuilder<InitProviderResult>();
var session = await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
var (handles, missing) = ResolveConnectors(options.Providers);
foreach (var providerId in missing)
{
@@ -100,15 +95,15 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
try
{
await ValidateConnectorAsync(handle, cancellationToken).ConfigureAwait(false);
await EnsureProviderRegistrationAsync(handle.Descriptor, session, cancellationToken).ConfigureAwait(false);
await EnsureProviderRegistrationAsync(handle.Descriptor, cancellationToken).ConfigureAwait(false);
stopwatch.Stop();
results.Add(new InitProviderResult(
handle.Descriptor.Id,
handle.Descriptor.DisplayName,
"succeeded",
stopwatch.Elapsed,
Error: null));
results.Add(new InitProviderResult(
handle.Descriptor.Id,
handle.Descriptor.DisplayName,
"succeeded",
stopwatch.Elapsed,
Error: null));
_logger.LogInformation("Excititor init validated provider {ProviderId} in {Duration}ms.", handle.Descriptor.Id, stopwatch.Elapsed.TotalMilliseconds);
}
@@ -148,8 +143,6 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
var startedAt = _timeProvider.GetUtcNow();
var since = ResolveSince(options.Since, options.Window, startedAt);
var results = ImmutableArray.CreateBuilder<ProviderRunResult>();
var session = await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
var (handles, missing) = ResolveConnectors(options.Providers);
foreach (var providerId in missing)
{
@@ -158,7 +151,7 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
foreach (var handle in handles)
{
var result = await ExecuteRunAsync(runId, handle, since, options.Force, session, cancellationToken).ConfigureAwait(false);
var result = await ExecuteRunAsync(runId, handle, since, options.Force, session, cancellationToken).ConfigureAwait(false);
results.Add(result);
}
@@ -173,20 +166,18 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
var runId = Guid.NewGuid();
var startedAt = _timeProvider.GetUtcNow();
var results = ImmutableArray.CreateBuilder<ProviderRunResult>();
var session = await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
var (handles, missing) = ResolveConnectors(options.Providers);
foreach (var providerId in missing)
{
results.Add(ProviderRunResult.Missing(providerId, since: null));
}
foreach (var handle in handles)
{
var since = await ResolveResumeSinceAsync(handle.Descriptor.Id, options.Checkpoint, session, cancellationToken).ConfigureAwait(false);
var result = await ExecuteRunAsync(runId, handle, since, force: false, session, cancellationToken).ConfigureAwait(false);
results.Add(result);
}
foreach (var handle in handles)
{
var since = await ResolveResumeSinceAsync(handle.Descriptor.Id, options.Checkpoint, session, cancellationToken).ConfigureAwait(false);
var result = await ExecuteRunAsync(runId, handle, since, force: false, session, cancellationToken).ConfigureAwait(false);
results.Add(result);
}
var completedAt = _timeProvider.GetUtcNow();
return new IngestRunSummary(runId, startedAt, completedAt, results.ToImmutable());
@@ -200,8 +191,6 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
var startedAt = _timeProvider.GetUtcNow();
var threshold = options.MaxAge is null ? (DateTimeOffset?)null : startedAt - options.MaxAge.Value;
var results = ImmutableArray.CreateBuilder<ReconcileProviderResult>();
var session = await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
var (handles, missing) = ResolveConnectors(options.Providers);
foreach (var providerId in missing)
{
@@ -219,8 +208,8 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
if (stale || state is null)
{
var since = stale ? threshold : lastUpdated;
var result = await ExecuteRunAsync(runId, handle, since, force: false, session, cancellationToken).ConfigureAwait(false);
results.Add(new ReconcileProviderResult(
var result = await ExecuteRunAsync(runId, handle, since, force: false, session, cancellationToken).ConfigureAwait(false);
results.Add(new ReconcileProviderResult(
handle.Descriptor.Id,
result.Status,
"reconciled",
@@ -232,15 +221,15 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
}
else
{
results.Add(new ReconcileProviderResult(
handle.Descriptor.Id,
"succeeded",
"skipped",
lastUpdated,
threshold,
Documents: 0,
Claims: 0,
Error: null));
results.Add(new ReconcileProviderResult(
handle.Descriptor.Id,
"succeeded",
"skipped",
lastUpdated,
threshold,
Documents: 0,
Claims: 0,
Error: null));
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
@@ -280,7 +269,7 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
await handle.Connector.ValidateAsync(VexConnectorSettings.Empty, cancellationToken).ConfigureAwait(false);
}
private async Task EnsureProviderRegistrationAsync(VexConnectorDescriptor descriptor, IClientSessionHandle session, CancellationToken cancellationToken)
private async Task EnsureProviderRegistrationAsync(VexConnectorDescriptor descriptor, CancellationToken cancellationToken)
{
var existing = await _providerStore.FindAsync(descriptor.Id, cancellationToken, session).ConfigureAwait(false);
if (existing is not null)
@@ -292,48 +281,48 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
await _providerStore.SaveAsync(provider, cancellationToken, session).ConfigureAwait(false);
}
private async Task<ProviderRunResult> ExecuteRunAsync(
Guid runId,
ConnectorHandle handle,
DateTimeOffset? since,
bool force,
IClientSessionHandle session,
CancellationToken cancellationToken)
{
var providerId = handle.Descriptor.Id;
var startedAt = _timeProvider.GetUtcNow();
var stopwatch = Stopwatch.StartNew();
using var scope = _logger.BeginScope(new Dictionary<string, object?>(StringComparer.Ordinal)
{
["tenant"] = _defaultTenant,
["runId"] = runId,
["providerId"] = providerId,
["window.since"] = since?.ToString("O", CultureInfo.InvariantCulture),
["force"] = force,
});
private async Task<ProviderRunResult> ExecuteRunAsync(
Guid runId,
ConnectorHandle handle,
DateTimeOffset? since,
bool force,
IClientSessionHandle session,
CancellationToken cancellationToken)
{
var providerId = handle.Descriptor.Id;
var startedAt = _timeProvider.GetUtcNow();
var stopwatch = Stopwatch.StartNew();
using var scope = _logger.BeginScope(new Dictionary<string, object?>(StringComparer.Ordinal)
{
["tenant"] = _defaultTenant,
["runId"] = runId,
["providerId"] = providerId,
["window.since"] = since?.ToString("O", CultureInfo.InvariantCulture),
["force"] = force,
});
try
{
await ValidateConnectorAsync(handle, cancellationToken).ConfigureAwait(false);
await EnsureProviderRegistrationAsync(handle.Descriptor, session, cancellationToken).ConfigureAwait(false);
if (force)
{
var resetState = new VexConnectorState(providerId, null, ImmutableArray<string>.Empty);
await _stateRepository.SaveAsync(resetState, cancellationToken, session).ConfigureAwait(false);
}
var stateBeforeRun = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false);
var resumeTokens = stateBeforeRun?.ResumeTokens ?? ImmutableDictionary<string, string>.Empty;
var context = new VexConnectorContext(
since,
VexConnectorSettings.Empty,
_rawStore,
_signatureVerifier,
_normalizerRouter,
_serviceProvider,
resumeTokens);
if (force)
{
var resetState = new VexConnectorState(providerId, null, ImmutableArray<string>.Empty);
await _stateRepository.SaveAsync(resetState, cancellationToken, session).ConfigureAwait(false);
}
var stateBeforeRun = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false);
var resumeTokens = stateBeforeRun?.ResumeTokens ?? ImmutableDictionary<string, string>.Empty;
var context = new VexConnectorContext(
since,
VexConnectorSettings.Empty,
_rawStore,
_signatureVerifier,
_normalizerRouter,
_serviceProvider,
resumeTokens);
var documents = 0;
var claims = 0;
@@ -354,25 +343,25 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
stopwatch.Stop();
var completedAt = _timeProvider.GetUtcNow();
var stateAfterRun = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false);
var checkpoint = stateAfterRun?.DocumentDigests.IsDefaultOrEmpty == false
? stateAfterRun.DocumentDigests[^1]
: lastDigest;
var result = new ProviderRunResult(
providerId,
"succeeded",
var stateAfterRun = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false);
var checkpoint = stateAfterRun?.DocumentDigests.IsDefaultOrEmpty == false
? stateAfterRun.DocumentDigests[^1]
: lastDigest;
var result = new ProviderRunResult(
providerId,
"succeeded",
documents,
claims,
startedAt,
completedAt,
stopwatch.Elapsed,
lastDigest,
stateAfterRun?.LastUpdated,
checkpoint,
null,
since);
lastDigest,
stateAfterRun?.LastUpdated,
checkpoint,
null,
since);
_logger.LogInformation(
"Excititor ingest provider {ProviderId} completed: documents={Documents} claims={Claims} since={Since} duration={Duration}ms",

2
src/Excititor/StellaOps.Excititor.WebService/Services/VexObservationProjectionService.cs
View File

@@ -6,7 +6,7 @@ using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.Core.Storage;
namespace StellaOps.Excititor.WebService.Services;