up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
This commit is contained in:
@@ -1,287 +1,287 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Routing;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Endpoints;
|
||||
|
||||
internal static class IngestEndpoints
|
||||
{
|
||||
private const string AdminScope = "vex.admin";
|
||||
|
||||
public static void MapIngestEndpoints(IEndpointRouteBuilder app)
|
||||
{
|
||||
var group = app.MapGroup("/excititor");
|
||||
|
||||
group.MapPost("/init", HandleInitAsync);
|
||||
group.MapPost("/ingest/run", HandleRunAsync);
|
||||
group.MapPost("/ingest/resume", HandleResumeAsync);
|
||||
group.MapPost("/reconcile", HandleReconcileAsync);
|
||||
}
|
||||
|
||||
internal static async Task<IResult> HandleInitAsync(
|
||||
HttpContext httpContext,
|
||||
ExcititorInitRequest request,
|
||||
IVexIngestOrchestrator orchestrator,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
var providerIds = NormalizeProviders(request.Providers);
|
||||
_ = timeProvider;
|
||||
var options = new IngestInitOptions(providerIds, request.Resume ?? false);
|
||||
|
||||
var summary = await orchestrator.InitializeAsync(options, cancellationToken).ConfigureAwait(false);
|
||||
var message = $"Initialized {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed.";
|
||||
|
||||
return TypedResults.Ok<object>(new
|
||||
{
|
||||
message,
|
||||
runId = summary.RunId,
|
||||
startedAt = summary.StartedAt,
|
||||
completedAt = summary.CompletedAt,
|
||||
providers = summary.Providers.Select(static provider => new
|
||||
{
|
||||
providerId = provider.ProviderId,
|
||||
displayName = provider.DisplayName,
|
||||
status = provider.Status,
|
||||
durationMs = provider.Duration.TotalMilliseconds,
|
||||
error = provider.Error
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
internal static async Task<IResult> HandleRunAsync(
|
||||
HttpContext httpContext,
|
||||
ExcititorIngestRunRequest request,
|
||||
IVexIngestOrchestrator orchestrator,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryParseDateTimeOffset(request.Since, out var since, out var sinceError))
|
||||
{
|
||||
return TypedResults.BadRequest<object>(new { message = sinceError });
|
||||
}
|
||||
|
||||
if (!TryParseTimeSpan(request.Window, out var window, out var windowError))
|
||||
{
|
||||
return TypedResults.BadRequest<object>(new { message = windowError });
|
||||
}
|
||||
|
||||
_ = timeProvider;
|
||||
var providerIds = NormalizeProviders(request.Providers);
|
||||
var options = new IngestRunOptions(
|
||||
providerIds,
|
||||
since,
|
||||
window,
|
||||
request.Force ?? false);
|
||||
|
||||
var summary = await orchestrator.RunAsync(options, cancellationToken).ConfigureAwait(false);
|
||||
var message = $"Ingest run completed for {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed.";
|
||||
|
||||
return TypedResults.Ok<object>(new
|
||||
{
|
||||
message,
|
||||
runId = summary.RunId,
|
||||
startedAt = summary.StartedAt,
|
||||
completedAt = summary.CompletedAt,
|
||||
durationMs = summary.Duration.TotalMilliseconds,
|
||||
providers = summary.Providers.Select(static provider => new
|
||||
{
|
||||
providerId = provider.ProviderId,
|
||||
status = provider.Status,
|
||||
documents = provider.Documents,
|
||||
claims = provider.Claims,
|
||||
startedAt = provider.StartedAt,
|
||||
completedAt = provider.CompletedAt,
|
||||
durationMs = provider.Duration.TotalMilliseconds,
|
||||
lastDigest = provider.LastDigest,
|
||||
lastUpdated = provider.LastUpdated,
|
||||
checkpoint = provider.Checkpoint,
|
||||
error = provider.Error
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
internal static async Task<IResult> HandleResumeAsync(
|
||||
HttpContext httpContext,
|
||||
ExcititorIngestResumeRequest request,
|
||||
IVexIngestOrchestrator orchestrator,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
_ = timeProvider;
|
||||
var providerIds = NormalizeProviders(request.Providers);
|
||||
var options = new IngestResumeOptions(providerIds, request.Checkpoint);
|
||||
|
||||
var summary = await orchestrator.ResumeAsync(options, cancellationToken).ConfigureAwait(false);
|
||||
var message = $"Resume run completed for {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed.";
|
||||
|
||||
return TypedResults.Ok<object>(new
|
||||
{
|
||||
message,
|
||||
runId = summary.RunId,
|
||||
startedAt = summary.StartedAt,
|
||||
completedAt = summary.CompletedAt,
|
||||
durationMs = summary.Duration.TotalMilliseconds,
|
||||
providers = summary.Providers.Select(static provider => new
|
||||
{
|
||||
providerId = provider.ProviderId,
|
||||
status = provider.Status,
|
||||
documents = provider.Documents,
|
||||
claims = provider.Claims,
|
||||
startedAt = provider.StartedAt,
|
||||
completedAt = provider.CompletedAt,
|
||||
durationMs = provider.Duration.TotalMilliseconds,
|
||||
since = provider.Since,
|
||||
checkpoint = provider.Checkpoint,
|
||||
error = provider.Error
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
internal static async Task<IResult> HandleReconcileAsync(
|
||||
HttpContext httpContext,
|
||||
ExcititorReconcileRequest request,
|
||||
IVexIngestOrchestrator orchestrator,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryParseTimeSpan(request.MaxAge, out var maxAge, out var error))
|
||||
{
|
||||
return TypedResults.BadRequest<object>(new { message = error });
|
||||
}
|
||||
|
||||
_ = timeProvider;
|
||||
var providerIds = NormalizeProviders(request.Providers);
|
||||
var options = new ReconcileOptions(providerIds, maxAge);
|
||||
|
||||
var summary = await orchestrator.ReconcileAsync(options, cancellationToken).ConfigureAwait(false);
|
||||
var message = $"Reconcile completed for {summary.ProviderCount} provider(s); {summary.ReconciledCount} reconciled, {summary.SkippedCount} skipped, {summary.FailureCount} failed.";
|
||||
|
||||
return TypedResults.Ok<object>(new
|
||||
{
|
||||
message,
|
||||
runId = summary.RunId,
|
||||
startedAt = summary.StartedAt,
|
||||
completedAt = summary.CompletedAt,
|
||||
durationMs = summary.Duration.TotalMilliseconds,
|
||||
providers = summary.Providers.Select(static provider => new
|
||||
{
|
||||
providerId = provider.ProviderId,
|
||||
status = provider.Status,
|
||||
action = provider.Action,
|
||||
lastUpdated = provider.LastUpdated,
|
||||
threshold = provider.Threshold,
|
||||
documents = provider.Documents,
|
||||
claims = provider.Claims,
|
||||
error = provider.Error
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
internal static ImmutableArray<string> NormalizeProviders(IReadOnlyCollection<string>? providers)
|
||||
{
|
||||
if (providers is null || providers.Count == 0)
|
||||
{
|
||||
return ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
var set = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var provider in providers)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(provider))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
set.Add(provider.Trim());
|
||||
}
|
||||
|
||||
return set.ToImmutableArray();
|
||||
}
|
||||
|
||||
internal static bool TryParseDateTimeOffset(string? value, out DateTimeOffset? result, out string? error)
|
||||
{
|
||||
result = null;
|
||||
error = null;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (DateTimeOffset.TryParse(
|
||||
value.Trim(),
|
||||
CultureInfo.InvariantCulture,
|
||||
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
|
||||
out var parsed))
|
||||
{
|
||||
result = parsed;
|
||||
return true;
|
||||
}
|
||||
|
||||
error = "Invalid 'since' value. Use ISO-8601 format (e.g. 2025-10-19T12:30:00Z).";
|
||||
return false;
|
||||
}
|
||||
|
||||
internal static bool TryParseTimeSpan(string? value, out TimeSpan? result, out string? error)
|
||||
{
|
||||
result = null;
|
||||
error = null;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TimeSpan.TryParse(value.Trim(), CultureInfo.InvariantCulture, out var parsed) && parsed >= TimeSpan.Zero)
|
||||
{
|
||||
result = parsed;
|
||||
return true;
|
||||
}
|
||||
|
||||
error = "Invalid duration value. Use TimeSpan format (e.g. 1.00:00:00).";
|
||||
return false;
|
||||
}
|
||||
|
||||
internal sealed record ExcititorInitRequest(IReadOnlyList<string>? Providers, bool? Resume);
|
||||
|
||||
internal sealed record ExcititorIngestRunRequest(
|
||||
IReadOnlyList<string>? Providers,
|
||||
string? Since,
|
||||
string? Window,
|
||||
bool? Force);
|
||||
|
||||
internal sealed record ExcititorIngestResumeRequest(
|
||||
IReadOnlyList<string>? Providers,
|
||||
string? Checkpoint);
|
||||
|
||||
internal sealed record ExcititorReconcileRequest(
|
||||
IReadOnlyList<string>? Providers,
|
||||
string? MaxAge);
|
||||
}
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Routing;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Endpoints;
|
||||
|
||||
internal static class IngestEndpoints
|
||||
{
|
||||
private const string AdminScope = "vex.admin";
|
||||
|
||||
public static void MapIngestEndpoints(IEndpointRouteBuilder app)
|
||||
{
|
||||
var group = app.MapGroup("/excititor");
|
||||
|
||||
group.MapPost("/init", HandleInitAsync);
|
||||
group.MapPost("/ingest/run", HandleRunAsync);
|
||||
group.MapPost("/ingest/resume", HandleResumeAsync);
|
||||
group.MapPost("/reconcile", HandleReconcileAsync);
|
||||
}
|
||||
|
||||
internal static async Task<IResult> HandleInitAsync(
|
||||
HttpContext httpContext,
|
||||
ExcititorInitRequest request,
|
||||
IVexIngestOrchestrator orchestrator,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
var providerIds = NormalizeProviders(request.Providers);
|
||||
_ = timeProvider;
|
||||
var options = new IngestInitOptions(providerIds, request.Resume ?? false);
|
||||
|
||||
var summary = await orchestrator.InitializeAsync(options, cancellationToken).ConfigureAwait(false);
|
||||
var message = $"Initialized {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed.";
|
||||
|
||||
return TypedResults.Ok<object>(new
|
||||
{
|
||||
message,
|
||||
runId = summary.RunId,
|
||||
startedAt = summary.StartedAt,
|
||||
completedAt = summary.CompletedAt,
|
||||
providers = summary.Providers.Select(static provider => new
|
||||
{
|
||||
providerId = provider.ProviderId,
|
||||
displayName = provider.DisplayName,
|
||||
status = provider.Status,
|
||||
durationMs = provider.Duration.TotalMilliseconds,
|
||||
error = provider.Error
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
internal static async Task<IResult> HandleRunAsync(
|
||||
HttpContext httpContext,
|
||||
ExcititorIngestRunRequest request,
|
||||
IVexIngestOrchestrator orchestrator,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryParseDateTimeOffset(request.Since, out var since, out var sinceError))
|
||||
{
|
||||
return TypedResults.BadRequest<object>(new { message = sinceError });
|
||||
}
|
||||
|
||||
if (!TryParseTimeSpan(request.Window, out var window, out var windowError))
|
||||
{
|
||||
return TypedResults.BadRequest<object>(new { message = windowError });
|
||||
}
|
||||
|
||||
_ = timeProvider;
|
||||
var providerIds = NormalizeProviders(request.Providers);
|
||||
var options = new IngestRunOptions(
|
||||
providerIds,
|
||||
since,
|
||||
window,
|
||||
request.Force ?? false);
|
||||
|
||||
var summary = await orchestrator.RunAsync(options, cancellationToken).ConfigureAwait(false);
|
||||
var message = $"Ingest run completed for {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed.";
|
||||
|
||||
return TypedResults.Ok<object>(new
|
||||
{
|
||||
message,
|
||||
runId = summary.RunId,
|
||||
startedAt = summary.StartedAt,
|
||||
completedAt = summary.CompletedAt,
|
||||
durationMs = summary.Duration.TotalMilliseconds,
|
||||
providers = summary.Providers.Select(static provider => new
|
||||
{
|
||||
providerId = provider.ProviderId,
|
||||
status = provider.Status,
|
||||
documents = provider.Documents,
|
||||
claims = provider.Claims,
|
||||
startedAt = provider.StartedAt,
|
||||
completedAt = provider.CompletedAt,
|
||||
durationMs = provider.Duration.TotalMilliseconds,
|
||||
lastDigest = provider.LastDigest,
|
||||
lastUpdated = provider.LastUpdated,
|
||||
checkpoint = provider.Checkpoint,
|
||||
error = provider.Error
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
internal static async Task<IResult> HandleResumeAsync(
|
||||
HttpContext httpContext,
|
||||
ExcititorIngestResumeRequest request,
|
||||
IVexIngestOrchestrator orchestrator,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
_ = timeProvider;
|
||||
var providerIds = NormalizeProviders(request.Providers);
|
||||
var options = new IngestResumeOptions(providerIds, request.Checkpoint);
|
||||
|
||||
var summary = await orchestrator.ResumeAsync(options, cancellationToken).ConfigureAwait(false);
|
||||
var message = $"Resume run completed for {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed.";
|
||||
|
||||
return TypedResults.Ok<object>(new
|
||||
{
|
||||
message,
|
||||
runId = summary.RunId,
|
||||
startedAt = summary.StartedAt,
|
||||
completedAt = summary.CompletedAt,
|
||||
durationMs = summary.Duration.TotalMilliseconds,
|
||||
providers = summary.Providers.Select(static provider => new
|
||||
{
|
||||
providerId = provider.ProviderId,
|
||||
status = provider.Status,
|
||||
documents = provider.Documents,
|
||||
claims = provider.Claims,
|
||||
startedAt = provider.StartedAt,
|
||||
completedAt = provider.CompletedAt,
|
||||
durationMs = provider.Duration.TotalMilliseconds,
|
||||
since = provider.Since,
|
||||
checkpoint = provider.Checkpoint,
|
||||
error = provider.Error
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
internal static async Task<IResult> HandleReconcileAsync(
|
||||
HttpContext httpContext,
|
||||
ExcititorReconcileRequest request,
|
||||
IVexIngestOrchestrator orchestrator,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryParseTimeSpan(request.MaxAge, out var maxAge, out var error))
|
||||
{
|
||||
return TypedResults.BadRequest<object>(new { message = error });
|
||||
}
|
||||
|
||||
_ = timeProvider;
|
||||
var providerIds = NormalizeProviders(request.Providers);
|
||||
var options = new ReconcileOptions(providerIds, maxAge);
|
||||
|
||||
var summary = await orchestrator.ReconcileAsync(options, cancellationToken).ConfigureAwait(false);
|
||||
var message = $"Reconcile completed for {summary.ProviderCount} provider(s); {summary.ReconciledCount} reconciled, {summary.SkippedCount} skipped, {summary.FailureCount} failed.";
|
||||
|
||||
return TypedResults.Ok<object>(new
|
||||
{
|
||||
message,
|
||||
runId = summary.RunId,
|
||||
startedAt = summary.StartedAt,
|
||||
completedAt = summary.CompletedAt,
|
||||
durationMs = summary.Duration.TotalMilliseconds,
|
||||
providers = summary.Providers.Select(static provider => new
|
||||
{
|
||||
providerId = provider.ProviderId,
|
||||
status = provider.Status,
|
||||
action = provider.Action,
|
||||
lastUpdated = provider.LastUpdated,
|
||||
threshold = provider.Threshold,
|
||||
documents = provider.Documents,
|
||||
claims = provider.Claims,
|
||||
error = provider.Error
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
internal static ImmutableArray<string> NormalizeProviders(IReadOnlyCollection<string>? providers)
|
||||
{
|
||||
if (providers is null || providers.Count == 0)
|
||||
{
|
||||
return ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
var set = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var provider in providers)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(provider))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
set.Add(provider.Trim());
|
||||
}
|
||||
|
||||
return set.ToImmutableArray();
|
||||
}
|
||||
|
||||
internal static bool TryParseDateTimeOffset(string? value, out DateTimeOffset? result, out string? error)
|
||||
{
|
||||
result = null;
|
||||
error = null;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (DateTimeOffset.TryParse(
|
||||
value.Trim(),
|
||||
CultureInfo.InvariantCulture,
|
||||
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
|
||||
out var parsed))
|
||||
{
|
||||
result = parsed;
|
||||
return true;
|
||||
}
|
||||
|
||||
error = "Invalid 'since' value. Use ISO-8601 format (e.g. 2025-10-19T12:30:00Z).";
|
||||
return false;
|
||||
}
|
||||
|
||||
internal static bool TryParseTimeSpan(string? value, out TimeSpan? result, out string? error)
|
||||
{
|
||||
result = null;
|
||||
error = null;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TimeSpan.TryParse(value.Trim(), CultureInfo.InvariantCulture, out var parsed) && parsed >= TimeSpan.Zero)
|
||||
{
|
||||
result = parsed;
|
||||
return true;
|
||||
}
|
||||
|
||||
error = "Invalid duration value. Use TimeSpan format (e.g. 1.00:00:00).";
|
||||
return false;
|
||||
}
|
||||
|
||||
internal sealed record ExcititorInitRequest(IReadOnlyList<string>? Providers, bool? Resume);
|
||||
|
||||
internal sealed record ExcititorIngestRunRequest(
|
||||
IReadOnlyList<string>? Providers,
|
||||
string? Since,
|
||||
string? Window,
|
||||
bool? Force);
|
||||
|
||||
internal sealed record ExcititorIngestResumeRequest(
|
||||
IReadOnlyList<string>? Providers,
|
||||
string? Checkpoint);
|
||||
|
||||
internal sealed record ExcititorReconcileRequest(
|
||||
IReadOnlyList<string>? Providers,
|
||||
string? MaxAge);
|
||||
}
|
||||
|
||||
@@ -1,391 +1,391 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Export;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Endpoints;
|
||||
|
||||
internal static class MirrorEndpoints
|
||||
{
|
||||
public static void MapMirrorEndpoints(WebApplication app)
|
||||
{
|
||||
var group = app.MapGroup("/excititor/mirror");
|
||||
|
||||
group.MapGet("/domains", HandleListDomainsAsync);
|
||||
group.MapGet("/domains/{domainId}", HandleDomainDetailAsync);
|
||||
group.MapGet("/domains/{domainId}/index", HandleDomainIndexAsync);
|
||||
group.MapGet("/domains/{domainId}/exports/{exportKey}", HandleExportMetadataAsync);
|
||||
group.MapGet("/domains/{domainId}/exports/{exportKey}/download", HandleExportDownloadAsync);
|
||||
}
|
||||
|
||||
private static async Task<IResult> HandleListDomainsAsync(
|
||||
HttpContext httpContext,
|
||||
IOptions<MirrorDistributionOptions> options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var domains = options.Value.Domains
|
||||
.Select(static domain => new MirrorDomainSummary(
|
||||
domain.Id,
|
||||
string.IsNullOrWhiteSpace(domain.DisplayName) ? domain.Id : domain.DisplayName,
|
||||
domain.RequireAuthentication,
|
||||
Math.Max(domain.MaxIndexRequestsPerHour, 0),
|
||||
Math.Max(domain.MaxDownloadRequestsPerHour, 0)))
|
||||
.ToArray();
|
||||
|
||||
await WriteJsonAsync(httpContext, new MirrorDomainListResponse(domains), StatusCodes.Status200OK, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
private static async Task<IResult> HandleDomainDetailAsync(
|
||||
string domainId,
|
||||
HttpContext httpContext,
|
||||
IOptions<MirrorDistributionOptions> options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryFindDomain(options.Value, domainId, out var domain))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
var response = new MirrorDomainDetail(
|
||||
domain.Id,
|
||||
string.IsNullOrWhiteSpace(domain.DisplayName) ? domain.Id : domain.DisplayName,
|
||||
domain.RequireAuthentication,
|
||||
Math.Max(domain.MaxIndexRequestsPerHour, 0),
|
||||
Math.Max(domain.MaxDownloadRequestsPerHour, 0),
|
||||
domain.Exports.Select(static export => export.Key).OrderBy(static key => key, StringComparer.Ordinal).ToImmutableArray());
|
||||
|
||||
await WriteJsonAsync(httpContext, response, StatusCodes.Status200OK, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
private static async Task<IResult> HandleDomainIndexAsync(
|
||||
string domainId,
|
||||
HttpContext httpContext,
|
||||
IOptions<MirrorDistributionOptions> options,
|
||||
[FromServices] MirrorRateLimiter rateLimiter,
|
||||
[FromServices] IVexExportStore exportStore,
|
||||
[FromServices] TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryFindDomain(options.Value, domainId, out var domain))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
if (domain.RequireAuthentication && (httpContext.User?.Identity?.IsAuthenticated is not true))
|
||||
{
|
||||
return Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!rateLimiter.TryAcquire(domain.Id, "index", Math.Max(domain.MaxIndexRequestsPerHour, 0), out var retryAfter))
|
||||
{
|
||||
if (retryAfter is { } retry)
|
||||
{
|
||||
httpContext.Response.Headers.RetryAfter = ((int)Math.Ceiling(retry.TotalSeconds)).ToString(CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
await WritePlainTextAsync(httpContext, "mirror index quota exceeded", StatusCodes.Status429TooManyRequests, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
var resolvedExports = new List<MirrorExportIndexEntry>();
|
||||
foreach (var exportOption in domain.Exports)
|
||||
{
|
||||
if (!MirrorExportPlanner.TryBuild(exportOption, out var plan, out var error))
|
||||
{
|
||||
resolvedExports.Add(new MirrorExportIndexEntry(
|
||||
exportOption.Key,
|
||||
null,
|
||||
null,
|
||||
exportOption.Format,
|
||||
null,
|
||||
null,
|
||||
0,
|
||||
null,
|
||||
null,
|
||||
error ?? "invalid_export_configuration"));
|
||||
continue;
|
||||
}
|
||||
|
||||
var manifest = await exportStore.FindAsync(plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (manifest is null)
|
||||
{
|
||||
resolvedExports.Add(new MirrorExportIndexEntry(
|
||||
exportOption.Key,
|
||||
null,
|
||||
plan.Signature.Value,
|
||||
plan.Format.ToString().ToLowerInvariant(),
|
||||
null,
|
||||
null,
|
||||
0,
|
||||
null,
|
||||
null,
|
||||
"manifest_not_found"));
|
||||
continue;
|
||||
}
|
||||
|
||||
resolvedExports.Add(new MirrorExportIndexEntry(
|
||||
exportOption.Key,
|
||||
manifest.ExportId,
|
||||
manifest.QuerySignature.Value,
|
||||
manifest.Format.ToString().ToLowerInvariant(),
|
||||
manifest.CreatedAt,
|
||||
manifest.Artifact,
|
||||
manifest.SizeBytes,
|
||||
manifest.ConsensusRevision,
|
||||
manifest.Attestation is null ? null : new MirrorExportAttestation(manifest.Attestation.PredicateType, manifest.Attestation.Rekor?.Location, manifest.Attestation.EnvelopeDigest, manifest.Attestation.SignedAt),
|
||||
null));
|
||||
}
|
||||
|
||||
var indexResponse = new MirrorDomainIndex(
|
||||
domain.Id,
|
||||
string.IsNullOrWhiteSpace(domain.DisplayName) ? domain.Id : domain.DisplayName,
|
||||
timeProvider.GetUtcNow(),
|
||||
resolvedExports.ToImmutableArray());
|
||||
|
||||
await WriteJsonAsync(httpContext, indexResponse, StatusCodes.Status200OK, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
private static async Task<IResult> HandleExportMetadataAsync(
|
||||
string domainId,
|
||||
string exportKey,
|
||||
HttpContext httpContext,
|
||||
IOptions<MirrorDistributionOptions> options,
|
||||
[FromServices] MirrorRateLimiter rateLimiter,
|
||||
[FromServices] IVexExportStore exportStore,
|
||||
[FromServices] TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryFindDomain(options.Value, domainId, out var domain))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
if (domain.RequireAuthentication && (httpContext.User?.Identity?.IsAuthenticated is not true))
|
||||
{
|
||||
return Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!TryFindExport(domain, exportKey, out var exportOptions))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
if (!MirrorExportPlanner.TryBuild(exportOptions, out var plan, out var error))
|
||||
{
|
||||
await WritePlainTextAsync(httpContext, error ?? "invalid_export_configuration", StatusCodes.Status503ServiceUnavailable, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
var manifest = await exportStore.FindAsync(plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false);
|
||||
if (manifest is null)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
var payload = new MirrorExportMetadata(
|
||||
domain.Id,
|
||||
exportOptions.Key,
|
||||
manifest.ExportId,
|
||||
manifest.QuerySignature.Value,
|
||||
manifest.Format.ToString().ToLowerInvariant(),
|
||||
manifest.CreatedAt,
|
||||
manifest.Artifact,
|
||||
manifest.SizeBytes,
|
||||
manifest.SourceProviders,
|
||||
manifest.Attestation is null ? null : new MirrorExportAttestation(manifest.Attestation.PredicateType, manifest.Attestation.Rekor?.Location, manifest.Attestation.EnvelopeDigest, manifest.Attestation.SignedAt));
|
||||
|
||||
await WriteJsonAsync(httpContext, payload, StatusCodes.Status200OK, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
private static async Task<IResult> HandleExportDownloadAsync(
|
||||
string domainId,
|
||||
string exportKey,
|
||||
HttpContext httpContext,
|
||||
IOptions<MirrorDistributionOptions> options,
|
||||
[FromServices] MirrorRateLimiter rateLimiter,
|
||||
[FromServices] IVexExportStore exportStore,
|
||||
[FromServices] IEnumerable<IVexArtifactStore> artifactStores,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryFindDomain(options.Value, domainId, out var domain))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
if (domain.RequireAuthentication && (httpContext.User?.Identity?.IsAuthenticated is not true))
|
||||
{
|
||||
return Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!rateLimiter.TryAcquire(domain.Id, "download", Math.Max(domain.MaxDownloadRequestsPerHour, 0), out var retryAfter))
|
||||
{
|
||||
if (retryAfter is { } retry)
|
||||
{
|
||||
httpContext.Response.Headers.RetryAfter = ((int)Math.Ceiling(retry.TotalSeconds)).ToString(CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
await WritePlainTextAsync(httpContext, "mirror download quota exceeded", StatusCodes.Status429TooManyRequests, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
if (!TryFindExport(domain, exportKey, out var exportOptions) || !MirrorExportPlanner.TryBuild(exportOptions, out var plan, out _))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
var manifest = await exportStore.FindAsync(plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false);
|
||||
if (manifest is null)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
Stream? contentStream = null;
|
||||
foreach (var store in artifactStores)
|
||||
{
|
||||
contentStream = await store.OpenReadAsync(manifest.Artifact, cancellationToken).ConfigureAwait(false);
|
||||
if (contentStream is not null)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (contentStream is null)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
await using (contentStream.ConfigureAwait(false))
|
||||
{
|
||||
var contentType = ResolveContentType(manifest.Format);
|
||||
httpContext.Response.StatusCode = StatusCodes.Status200OK;
|
||||
httpContext.Response.ContentType = contentType;
|
||||
httpContext.Response.Headers.ContentDisposition = FormattableString.Invariant($"attachment; filename=\"{BuildDownloadFileName(domain.Id, exportOptions.Key, manifest.Format)}\"");
|
||||
|
||||
await contentStream.CopyToAsync(httpContext.Response.Body, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
private static bool TryFindDomain(MirrorDistributionOptions options, string domainId, out MirrorDomainOptions domain)
|
||||
{
|
||||
domain = options.Domains.FirstOrDefault(d => string.Equals(d.Id, domainId, StringComparison.OrdinalIgnoreCase))!;
|
||||
return domain is not null;
|
||||
}
|
||||
|
||||
private static bool TryFindExport(MirrorDomainOptions domain, string exportKey, out MirrorExportOptions export)
|
||||
{
|
||||
export = domain.Exports.FirstOrDefault(e => string.Equals(e.Key, exportKey, StringComparison.OrdinalIgnoreCase))!;
|
||||
return export is not null;
|
||||
}
|
||||
|
||||
private static string ResolveContentType(VexExportFormat format)
|
||||
=> format switch
|
||||
{
|
||||
VexExportFormat.Json => "application/json",
|
||||
VexExportFormat.JsonLines => "application/jsonl",
|
||||
VexExportFormat.OpenVex => "application/json",
|
||||
VexExportFormat.Csaf => "application/json",
|
||||
VexExportFormat.CycloneDx => "application/json",
|
||||
_ => "application/octet-stream",
|
||||
};
|
||||
|
||||
private static string BuildDownloadFileName(string domainId, string exportKey, VexExportFormat format)
|
||||
{
|
||||
var builder = new StringBuilder(domainId.Length + exportKey.Length + 8);
|
||||
builder.Append(domainId).Append('-').Append(exportKey);
|
||||
builder.Append(format switch
|
||||
{
|
||||
VexExportFormat.Json => ".json",
|
||||
VexExportFormat.JsonLines => ".jsonl",
|
||||
VexExportFormat.OpenVex => ".openvex.json",
|
||||
VexExportFormat.Csaf => ".csaf.json",
|
||||
VexExportFormat.CycloneDx => ".cyclonedx.json",
|
||||
_ => ".bin",
|
||||
});
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
private static async Task WritePlainTextAsync(HttpContext context, string message, int statusCode, CancellationToken cancellationToken)
|
||||
{
|
||||
context.Response.StatusCode = statusCode;
|
||||
context.Response.ContentType = "text/plain";
|
||||
await context.Response.WriteAsync(message, cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task WriteJsonAsync<T>(HttpContext context, T payload, int statusCode, CancellationToken cancellationToken)
|
||||
{
|
||||
context.Response.StatusCode = statusCode;
|
||||
context.Response.ContentType = "application/json";
|
||||
var json = VexCanonicalJsonSerializer.Serialize(payload);
|
||||
await context.Response.WriteAsync(json, cancellationToken);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
internal sealed record MirrorDomainListResponse(IReadOnlyList<MirrorDomainSummary> Domains);
|
||||
|
||||
internal sealed record MirrorDomainSummary(
|
||||
string Id,
|
||||
string DisplayName,
|
||||
bool RequireAuthentication,
|
||||
int MaxIndexRequestsPerHour,
|
||||
int MaxDownloadRequestsPerHour);
|
||||
|
||||
internal sealed record MirrorDomainDetail(
|
||||
string Id,
|
||||
string DisplayName,
|
||||
bool RequireAuthentication,
|
||||
int MaxIndexRequestsPerHour,
|
||||
int MaxDownloadRequestsPerHour,
|
||||
IReadOnlyList<string> Exports);
|
||||
|
||||
internal sealed record MirrorDomainIndex(
|
||||
string Id,
|
||||
string DisplayName,
|
||||
DateTimeOffset GeneratedAt,
|
||||
IReadOnlyList<MirrorExportIndexEntry> Exports);
|
||||
|
||||
internal sealed record MirrorExportIndexEntry(
|
||||
string ExportKey,
|
||||
string? ExportId,
|
||||
string? QuerySignature,
|
||||
string Format,
|
||||
DateTimeOffset? CreatedAt,
|
||||
VexContentAddress? Artifact,
|
||||
long SizeBytes,
|
||||
string? ConsensusRevision,
|
||||
MirrorExportAttestation? Attestation,
|
||||
string? Status);
|
||||
|
||||
internal sealed record MirrorExportAttestation(
|
||||
string PredicateType,
|
||||
string? RekorLocation,
|
||||
string? EnvelopeDigest,
|
||||
DateTimeOffset? SignedAt);
|
||||
|
||||
internal sealed record MirrorExportMetadata(
|
||||
string DomainId,
|
||||
string ExportKey,
|
||||
string ExportId,
|
||||
string QuerySignature,
|
||||
string Format,
|
||||
DateTimeOffset CreatedAt,
|
||||
VexContentAddress Artifact,
|
||||
long SizeBytes,
|
||||
IReadOnlyList<string> SourceProviders,
|
||||
MirrorExportAttestation? Attestation);
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Export;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Endpoints;
|
||||
|
||||
internal static class MirrorEndpoints
|
||||
{
|
||||
public static void MapMirrorEndpoints(WebApplication app)
|
||||
{
|
||||
var group = app.MapGroup("/excititor/mirror");
|
||||
|
||||
group.MapGet("/domains", HandleListDomainsAsync);
|
||||
group.MapGet("/domains/{domainId}", HandleDomainDetailAsync);
|
||||
group.MapGet("/domains/{domainId}/index", HandleDomainIndexAsync);
|
||||
group.MapGet("/domains/{domainId}/exports/{exportKey}", HandleExportMetadataAsync);
|
||||
group.MapGet("/domains/{domainId}/exports/{exportKey}/download", HandleExportDownloadAsync);
|
||||
}
|
||||
|
||||
private static async Task<IResult> HandleListDomainsAsync(
|
||||
HttpContext httpContext,
|
||||
IOptions<MirrorDistributionOptions> options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var domains = options.Value.Domains
|
||||
.Select(static domain => new MirrorDomainSummary(
|
||||
domain.Id,
|
||||
string.IsNullOrWhiteSpace(domain.DisplayName) ? domain.Id : domain.DisplayName,
|
||||
domain.RequireAuthentication,
|
||||
Math.Max(domain.MaxIndexRequestsPerHour, 0),
|
||||
Math.Max(domain.MaxDownloadRequestsPerHour, 0)))
|
||||
.ToArray();
|
||||
|
||||
await WriteJsonAsync(httpContext, new MirrorDomainListResponse(domains), StatusCodes.Status200OK, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
private static async Task<IResult> HandleDomainDetailAsync(
|
||||
string domainId,
|
||||
HttpContext httpContext,
|
||||
IOptions<MirrorDistributionOptions> options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryFindDomain(options.Value, domainId, out var domain))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
var response = new MirrorDomainDetail(
|
||||
domain.Id,
|
||||
string.IsNullOrWhiteSpace(domain.DisplayName) ? domain.Id : domain.DisplayName,
|
||||
domain.RequireAuthentication,
|
||||
Math.Max(domain.MaxIndexRequestsPerHour, 0),
|
||||
Math.Max(domain.MaxDownloadRequestsPerHour, 0),
|
||||
domain.Exports.Select(static export => export.Key).OrderBy(static key => key, StringComparer.Ordinal).ToImmutableArray());
|
||||
|
||||
await WriteJsonAsync(httpContext, response, StatusCodes.Status200OK, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
private static async Task<IResult> HandleDomainIndexAsync(
|
||||
string domainId,
|
||||
HttpContext httpContext,
|
||||
IOptions<MirrorDistributionOptions> options,
|
||||
[FromServices] MirrorRateLimiter rateLimiter,
|
||||
[FromServices] IVexExportStore exportStore,
|
||||
[FromServices] TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryFindDomain(options.Value, domainId, out var domain))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
if (domain.RequireAuthentication && (httpContext.User?.Identity?.IsAuthenticated is not true))
|
||||
{
|
||||
return Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!rateLimiter.TryAcquire(domain.Id, "index", Math.Max(domain.MaxIndexRequestsPerHour, 0), out var retryAfter))
|
||||
{
|
||||
if (retryAfter is { } retry)
|
||||
{
|
||||
httpContext.Response.Headers.RetryAfter = ((int)Math.Ceiling(retry.TotalSeconds)).ToString(CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
await WritePlainTextAsync(httpContext, "mirror index quota exceeded", StatusCodes.Status429TooManyRequests, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
var resolvedExports = new List<MirrorExportIndexEntry>();
|
||||
foreach (var exportOption in domain.Exports)
|
||||
{
|
||||
if (!MirrorExportPlanner.TryBuild(exportOption, out var plan, out var error))
|
||||
{
|
||||
resolvedExports.Add(new MirrorExportIndexEntry(
|
||||
exportOption.Key,
|
||||
null,
|
||||
null,
|
||||
exportOption.Format,
|
||||
null,
|
||||
null,
|
||||
0,
|
||||
null,
|
||||
null,
|
||||
error ?? "invalid_export_configuration"));
|
||||
continue;
|
||||
}
|
||||
|
||||
var manifest = await exportStore.FindAsync(plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (manifest is null)
|
||||
{
|
||||
resolvedExports.Add(new MirrorExportIndexEntry(
|
||||
exportOption.Key,
|
||||
null,
|
||||
plan.Signature.Value,
|
||||
plan.Format.ToString().ToLowerInvariant(),
|
||||
null,
|
||||
null,
|
||||
0,
|
||||
null,
|
||||
null,
|
||||
"manifest_not_found"));
|
||||
continue;
|
||||
}
|
||||
|
||||
resolvedExports.Add(new MirrorExportIndexEntry(
|
||||
exportOption.Key,
|
||||
manifest.ExportId,
|
||||
manifest.QuerySignature.Value,
|
||||
manifest.Format.ToString().ToLowerInvariant(),
|
||||
manifest.CreatedAt,
|
||||
manifest.Artifact,
|
||||
manifest.SizeBytes,
|
||||
manifest.ConsensusRevision,
|
||||
manifest.Attestation is null ? null : new MirrorExportAttestation(manifest.Attestation.PredicateType, manifest.Attestation.Rekor?.Location, manifest.Attestation.EnvelopeDigest, manifest.Attestation.SignedAt),
|
||||
null));
|
||||
}
|
||||
|
||||
var indexResponse = new MirrorDomainIndex(
|
||||
domain.Id,
|
||||
string.IsNullOrWhiteSpace(domain.DisplayName) ? domain.Id : domain.DisplayName,
|
||||
timeProvider.GetUtcNow(),
|
||||
resolvedExports.ToImmutableArray());
|
||||
|
||||
await WriteJsonAsync(httpContext, indexResponse, StatusCodes.Status200OK, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
private static async Task<IResult> HandleExportMetadataAsync(
|
||||
string domainId,
|
||||
string exportKey,
|
||||
HttpContext httpContext,
|
||||
IOptions<MirrorDistributionOptions> options,
|
||||
[FromServices] MirrorRateLimiter rateLimiter,
|
||||
[FromServices] IVexExportStore exportStore,
|
||||
[FromServices] TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryFindDomain(options.Value, domainId, out var domain))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
if (domain.RequireAuthentication && (httpContext.User?.Identity?.IsAuthenticated is not true))
|
||||
{
|
||||
return Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!TryFindExport(domain, exportKey, out var exportOptions))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
if (!MirrorExportPlanner.TryBuild(exportOptions, out var plan, out var error))
|
||||
{
|
||||
await WritePlainTextAsync(httpContext, error ?? "invalid_export_configuration", StatusCodes.Status503ServiceUnavailable, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
var manifest = await exportStore.FindAsync(plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false);
|
||||
if (manifest is null)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
var payload = new MirrorExportMetadata(
|
||||
domain.Id,
|
||||
exportOptions.Key,
|
||||
manifest.ExportId,
|
||||
manifest.QuerySignature.Value,
|
||||
manifest.Format.ToString().ToLowerInvariant(),
|
||||
manifest.CreatedAt,
|
||||
manifest.Artifact,
|
||||
manifest.SizeBytes,
|
||||
manifest.SourceProviders,
|
||||
manifest.Attestation is null ? null : new MirrorExportAttestation(manifest.Attestation.PredicateType, manifest.Attestation.Rekor?.Location, manifest.Attestation.EnvelopeDigest, manifest.Attestation.SignedAt));
|
||||
|
||||
await WriteJsonAsync(httpContext, payload, StatusCodes.Status200OK, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
private static async Task<IResult> HandleExportDownloadAsync(
|
||||
string domainId,
|
||||
string exportKey,
|
||||
HttpContext httpContext,
|
||||
IOptions<MirrorDistributionOptions> options,
|
||||
[FromServices] MirrorRateLimiter rateLimiter,
|
||||
[FromServices] IVexExportStore exportStore,
|
||||
[FromServices] IEnumerable<IVexArtifactStore> artifactStores,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryFindDomain(options.Value, domainId, out var domain))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
if (domain.RequireAuthentication && (httpContext.User?.Identity?.IsAuthenticated is not true))
|
||||
{
|
||||
return Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!rateLimiter.TryAcquire(domain.Id, "download", Math.Max(domain.MaxDownloadRequestsPerHour, 0), out var retryAfter))
|
||||
{
|
||||
if (retryAfter is { } retry)
|
||||
{
|
||||
httpContext.Response.Headers.RetryAfter = ((int)Math.Ceiling(retry.TotalSeconds)).ToString(CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
await WritePlainTextAsync(httpContext, "mirror download quota exceeded", StatusCodes.Status429TooManyRequests, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
if (!TryFindExport(domain, exportKey, out var exportOptions) || !MirrorExportPlanner.TryBuild(exportOptions, out var plan, out _))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
var manifest = await exportStore.FindAsync(plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false);
|
||||
if (manifest is null)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
Stream? contentStream = null;
|
||||
foreach (var store in artifactStores)
|
||||
{
|
||||
contentStream = await store.OpenReadAsync(manifest.Artifact, cancellationToken).ConfigureAwait(false);
|
||||
if (contentStream is not null)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (contentStream is null)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
await using (contentStream.ConfigureAwait(false))
|
||||
{
|
||||
var contentType = ResolveContentType(manifest.Format);
|
||||
httpContext.Response.StatusCode = StatusCodes.Status200OK;
|
||||
httpContext.Response.ContentType = contentType;
|
||||
httpContext.Response.Headers.ContentDisposition = FormattableString.Invariant($"attachment; filename=\"{BuildDownloadFileName(domain.Id, exportOptions.Key, manifest.Format)}\"");
|
||||
|
||||
await contentStream.CopyToAsync(httpContext.Response.Body, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
private static bool TryFindDomain(MirrorDistributionOptions options, string domainId, out MirrorDomainOptions domain)
|
||||
{
|
||||
domain = options.Domains.FirstOrDefault(d => string.Equals(d.Id, domainId, StringComparison.OrdinalIgnoreCase))!;
|
||||
return domain is not null;
|
||||
}
|
||||
|
||||
private static bool TryFindExport(MirrorDomainOptions domain, string exportKey, out MirrorExportOptions export)
|
||||
{
|
||||
export = domain.Exports.FirstOrDefault(e => string.Equals(e.Key, exportKey, StringComparison.OrdinalIgnoreCase))!;
|
||||
return export is not null;
|
||||
}
|
||||
|
||||
private static string ResolveContentType(VexExportFormat format)
|
||||
=> format switch
|
||||
{
|
||||
VexExportFormat.Json => "application/json",
|
||||
VexExportFormat.JsonLines => "application/jsonl",
|
||||
VexExportFormat.OpenVex => "application/json",
|
||||
VexExportFormat.Csaf => "application/json",
|
||||
VexExportFormat.CycloneDx => "application/json",
|
||||
_ => "application/octet-stream",
|
||||
};
|
||||
|
||||
private static string BuildDownloadFileName(string domainId, string exportKey, VexExportFormat format)
|
||||
{
|
||||
var builder = new StringBuilder(domainId.Length + exportKey.Length + 8);
|
||||
builder.Append(domainId).Append('-').Append(exportKey);
|
||||
builder.Append(format switch
|
||||
{
|
||||
VexExportFormat.Json => ".json",
|
||||
VexExportFormat.JsonLines => ".jsonl",
|
||||
VexExportFormat.OpenVex => ".openvex.json",
|
||||
VexExportFormat.Csaf => ".csaf.json",
|
||||
VexExportFormat.CycloneDx => ".cyclonedx.json",
|
||||
_ => ".bin",
|
||||
});
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
private static async Task WritePlainTextAsync(HttpContext context, string message, int statusCode, CancellationToken cancellationToken)
|
||||
{
|
||||
context.Response.StatusCode = statusCode;
|
||||
context.Response.ContentType = "text/plain";
|
||||
await context.Response.WriteAsync(message, cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task WriteJsonAsync<T>(HttpContext context, T payload, int statusCode, CancellationToken cancellationToken)
|
||||
{
|
||||
context.Response.StatusCode = statusCode;
|
||||
context.Response.ContentType = "application/json";
|
||||
var json = VexCanonicalJsonSerializer.Serialize(payload);
|
||||
await context.Response.WriteAsync(json, cancellationToken);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
internal sealed record MirrorDomainListResponse(IReadOnlyList<MirrorDomainSummary> Domains);
|
||||
|
||||
internal sealed record MirrorDomainSummary(
|
||||
string Id,
|
||||
string DisplayName,
|
||||
bool RequireAuthentication,
|
||||
int MaxIndexRequestsPerHour,
|
||||
int MaxDownloadRequestsPerHour);
|
||||
|
||||
internal sealed record MirrorDomainDetail(
|
||||
string Id,
|
||||
string DisplayName,
|
||||
bool RequireAuthentication,
|
||||
int MaxIndexRequestsPerHour,
|
||||
int MaxDownloadRequestsPerHour,
|
||||
IReadOnlyList<string> Exports);
|
||||
|
||||
internal sealed record MirrorDomainIndex(
|
||||
string Id,
|
||||
string DisplayName,
|
||||
DateTimeOffset GeneratedAt,
|
||||
IReadOnlyList<MirrorExportIndexEntry> Exports);
|
||||
|
||||
internal sealed record MirrorExportIndexEntry(
|
||||
string ExportKey,
|
||||
string? ExportId,
|
||||
string? QuerySignature,
|
||||
string Format,
|
||||
DateTimeOffset? CreatedAt,
|
||||
VexContentAddress? Artifact,
|
||||
long SizeBytes,
|
||||
string? ConsensusRevision,
|
||||
MirrorExportAttestation? Attestation,
|
||||
string? Status);
|
||||
|
||||
internal sealed record MirrorExportAttestation(
|
||||
string PredicateType,
|
||||
string? RekorLocation,
|
||||
string? EnvelopeDigest,
|
||||
DateTimeOffset? SignedAt);
|
||||
|
||||
internal sealed record MirrorExportMetadata(
|
||||
string DomainId,
|
||||
string ExportKey,
|
||||
string ExportId,
|
||||
string QuerySignature,
|
||||
string Format,
|
||||
DateTimeOffset CreatedAt,
|
||||
VexContentAddress Artifact,
|
||||
long SizeBytes,
|
||||
IReadOnlyList<string> SourceProviders,
|
||||
MirrorExportAttestation? Attestation);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,4 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Excititor.WebService.Tests")]
|
||||
[assembly: InternalsVisibleTo("StellaOps.Excititor.Core.UnitTests")]
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Excititor.WebService.Tests")]
|
||||
[assembly: InternalsVisibleTo("StellaOps.Excititor.Core.UnitTests")]
|
||||
|
||||
@@ -1,57 +1,57 @@
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Services;
|
||||
|
||||
internal sealed class MirrorRateLimiter
|
||||
{
|
||||
private readonly IMemoryCache _cache;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private static readonly TimeSpan Window = TimeSpan.FromHours(1);
|
||||
|
||||
public MirrorRateLimiter(IMemoryCache cache, TimeProvider timeProvider)
|
||||
{
|
||||
_cache = cache ?? throw new ArgumentNullException(nameof(cache));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
public bool TryAcquire(string domainId, string scope, int limit, out TimeSpan? retryAfter)
|
||||
{
|
||||
retryAfter = null;
|
||||
|
||||
if (limit <= 0 || limit == int.MaxValue)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var key = CreateKey(domainId, scope);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
var counter = _cache.Get<Counter>(key);
|
||||
if (counter is null || now - counter.WindowStart >= Window)
|
||||
{
|
||||
counter = new Counter(now, 0);
|
||||
}
|
||||
|
||||
if (counter.Count >= limit)
|
||||
{
|
||||
var windowEnd = counter.WindowStart + Window;
|
||||
retryAfter = windowEnd > now ? windowEnd - now : TimeSpan.Zero;
|
||||
return false;
|
||||
}
|
||||
|
||||
counter = counter with { Count = counter.Count + 1 };
|
||||
var absoluteExpiration = counter.WindowStart + Window;
|
||||
_cache.Set(key, counter, absoluteExpiration);
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string CreateKey(string domainId, string scope)
|
||||
=> string.Create(domainId.Length + scope.Length + 1, (domainId, scope), static (span, state) =>
|
||||
{
|
||||
state.domainId.AsSpan().CopyTo(span);
|
||||
span[state.domainId.Length] = '|';
|
||||
state.scope.AsSpan().CopyTo(span[(state.domainId.Length + 1)..]);
|
||||
});
|
||||
|
||||
private sealed record Counter(DateTimeOffset WindowStart, int Count);
|
||||
}
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Services;
|
||||
|
||||
internal sealed class MirrorRateLimiter
|
||||
{
|
||||
private readonly IMemoryCache _cache;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private static readonly TimeSpan Window = TimeSpan.FromHours(1);
|
||||
|
||||
public MirrorRateLimiter(IMemoryCache cache, TimeProvider timeProvider)
|
||||
{
|
||||
_cache = cache ?? throw new ArgumentNullException(nameof(cache));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
public bool TryAcquire(string domainId, string scope, int limit, out TimeSpan? retryAfter)
|
||||
{
|
||||
retryAfter = null;
|
||||
|
||||
if (limit <= 0 || limit == int.MaxValue)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var key = CreateKey(domainId, scope);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
var counter = _cache.Get<Counter>(key);
|
||||
if (counter is null || now - counter.WindowStart >= Window)
|
||||
{
|
||||
counter = new Counter(now, 0);
|
||||
}
|
||||
|
||||
if (counter.Count >= limit)
|
||||
{
|
||||
var windowEnd = counter.WindowStart + Window;
|
||||
retryAfter = windowEnd > now ? windowEnd - now : TimeSpan.Zero;
|
||||
return false;
|
||||
}
|
||||
|
||||
counter = counter with { Count = counter.Count + 1 };
|
||||
var absoluteExpiration = counter.WindowStart + Window;
|
||||
_cache.Set(key, counter, absoluteExpiration);
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string CreateKey(string domainId, string scope)
|
||||
=> string.Create(domainId.Length + scope.Length + 1, (domainId, scope), static (span, state) =>
|
||||
{
|
||||
state.domainId.AsSpan().CopyTo(span);
|
||||
span[state.domainId.Length] = '|';
|
||||
state.scope.AsSpan().CopyTo(span[(state.domainId.Length + 1)..]);
|
||||
});
|
||||
|
||||
private sealed record Counter(DateTimeOffset WindowStart, int Count);
|
||||
}
|
||||
|
||||
@@ -1,54 +1,54 @@
|
||||
using System.Linq;
|
||||
using System.Security.Claims;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Services;
|
||||
|
||||
internal static class ScopeAuthorization
|
||||
{
|
||||
public static IResult? RequireScope(HttpContext context, string scope)
|
||||
{
|
||||
if (context is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(context));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(scope))
|
||||
{
|
||||
throw new ArgumentException("Scope must be provided.", nameof(scope));
|
||||
}
|
||||
|
||||
var user = context.User;
|
||||
if (user?.Identity?.IsAuthenticated is not true)
|
||||
{
|
||||
return Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!HasScope(user, scope))
|
||||
{
|
||||
return Results.Forbid();
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool HasScope(ClaimsPrincipal user, string requiredScope)
|
||||
{
|
||||
var comparison = StringComparer.OrdinalIgnoreCase;
|
||||
foreach (var claim in user.FindAll("scope").Concat(user.FindAll("scp")))
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(claim.Value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var scopes = claim.Value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
if (scopes.Any(scope => comparison.Equals(scope, requiredScope)))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
using System.Linq;
|
||||
using System.Security.Claims;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Services;
|
||||
|
||||
internal static class ScopeAuthorization
|
||||
{
|
||||
public static IResult? RequireScope(HttpContext context, string scope)
|
||||
{
|
||||
if (context is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(context));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(scope))
|
||||
{
|
||||
throw new ArgumentException("Scope must be provided.", nameof(scope));
|
||||
}
|
||||
|
||||
var user = context.User;
|
||||
if (user?.Identity?.IsAuthenticated is not true)
|
||||
{
|
||||
return Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!HasScope(user, scope))
|
||||
{
|
||||
return Results.Forbid();
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool HasScope(ClaimsPrincipal user, string requiredScope)
|
||||
{
|
||||
var comparison = StringComparer.OrdinalIgnoreCase;
|
||||
foreach (var claim in user.FindAll("scope").Concat(user.FindAll("scp")))
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(claim.Value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var scopes = claim.Value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
if (scopes.Any(scope => comparison.Equals(scope, requiredScope)))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user