Add tests and implement StubBearer authentication for Signer endpoints

- Created SignerEndpointsTests to validate the SignDsse and VerifyReferrers endpoints.
- Implemented StubBearerAuthenticationDefaults and StubBearerAuthenticationHandler for token-based authentication.
- Developed ConcelierExporterClient for managing Trivy DB settings and export operations.
- Added TrivyDbSettingsPageComponent for UI interactions with Trivy DB settings, including form handling and export triggering.
- Implemented styles and HTML structure for Trivy DB settings page.
- Created NotifySmokeCheck tool for validating Redis event streams and Notify deliveries.
This commit is contained in:
master
2025-10-21 09:37:07 +03:00
parent d6cb41dd51
commit 48f3071e2a
298 changed files with 20490 additions and 5751 deletions

View File

@@ -20,48 +20,49 @@ internal static class IngestEndpoints
group.MapPost("/reconcile", HandleReconcileAsync);
}
private static async Task<IResult> HandleInitAsync(
HttpContext httpContext,
ExcititorInitRequest request,
IVexIngestOrchestrator orchestrator,
TimeProvider timeProvider,
CancellationToken cancellationToken)
internal static async Task<IResult> HandleInitAsync(
HttpContext httpContext,
ExcititorInitRequest request,
IVexIngestOrchestrator orchestrator,
TimeProvider timeProvider,
CancellationToken cancellationToken)
{
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
if (scopeResult is not null)
{
return scopeResult;
}
var providerIds = NormalizeProviders(request.Providers);
var options = new IngestInitOptions(providerIds, request.Resume ?? false, timeProvider);
{
return scopeResult;
}
var providerIds = NormalizeProviders(request.Providers);
_ = timeProvider;
var options = new IngestInitOptions(providerIds, request.Resume ?? false);
var summary = await orchestrator.InitializeAsync(options, cancellationToken).ConfigureAwait(false);
var message = $"Initialized {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed.";
return Results.Ok(new
{
message,
runId = summary.RunId,
startedAt = summary.StartedAt,
completedAt = summary.CompletedAt,
providers = summary.Providers.Select(static provider => new
{
provider.providerId,
provider.displayName,
provider.status,
provider.durationMs,
provider.error
})
});
}
return TypedResults.Ok<object>(new
{
message,
runId = summary.RunId,
startedAt = summary.StartedAt,
completedAt = summary.CompletedAt,
providers = summary.Providers.Select(static provider => new
{
providerId = provider.ProviderId,
displayName = provider.DisplayName,
status = provider.Status,
durationMs = provider.Duration.TotalMilliseconds,
error = provider.Error
})
});
}
private static async Task<IResult> HandleRunAsync(
HttpContext httpContext,
ExcititorIngestRunRequest request,
IVexIngestOrchestrator orchestrator,
TimeProvider timeProvider,
CancellationToken cancellationToken)
internal static async Task<IResult> HandleRunAsync(
HttpContext httpContext,
ExcititorIngestRunRequest request,
IVexIngestOrchestrator orchestrator,
TimeProvider timeProvider,
CancellationToken cancellationToken)
{
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
if (scopeResult is not null)
@@ -71,55 +72,98 @@ internal static class IngestEndpoints
if (!TryParseDateTimeOffset(request.Since, out var since, out var sinceError))
{
return Results.BadRequest(new { message = sinceError });
}
if (!TryParseTimeSpan(request.Window, out var window, out var windowError))
{
return Results.BadRequest(new { message = windowError });
}
var providerIds = NormalizeProviders(request.Providers);
var options = new IngestRunOptions(
providerIds,
since,
window,
request.Force ?? false,
timeProvider);
var summary = await orchestrator.RunAsync(options, cancellationToken).ConfigureAwait(false);
var message = $"Ingest run completed for {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed.";
return Results.Ok(new
{
message,
runId = summary.RunId,
startedAt = summary.StartedAt,
return TypedResults.BadRequest<object>(new { message = sinceError });
}
if (!TryParseTimeSpan(request.Window, out var window, out var windowError))
{
return TypedResults.BadRequest<object>(new { message = windowError });
}
_ = timeProvider;
var providerIds = NormalizeProviders(request.Providers);
var options = new IngestRunOptions(
providerIds,
since,
window,
request.Force ?? false);
var summary = await orchestrator.RunAsync(options, cancellationToken).ConfigureAwait(false);
var message = $"Ingest run completed for {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed.";
return TypedResults.Ok<object>(new
{
message,
runId = summary.RunId,
startedAt = summary.StartedAt,
completedAt = summary.CompletedAt,
durationMs = summary.Duration.TotalMilliseconds,
providers = summary.Providers.Select(static provider => new
{
provider.providerId,
provider.status,
provider.documents,
provider.claims,
provider.startedAt,
provider.completedAt,
provider.durationMs,
provider.lastDigest,
provider.lastUpdated,
provider.checkpoint,
provider.error
})
});
}
durationMs = summary.Duration.TotalMilliseconds,
providers = summary.Providers.Select(static provider => new
{
providerId = provider.ProviderId,
status = provider.Status,
documents = provider.Documents,
claims = provider.Claims,
startedAt = provider.StartedAt,
completedAt = provider.CompletedAt,
durationMs = provider.Duration.TotalMilliseconds,
lastDigest = provider.LastDigest,
lastUpdated = provider.LastUpdated,
checkpoint = provider.Checkpoint,
error = provider.Error
})
});
}
private static async Task<IResult> HandleResumeAsync(
HttpContext httpContext,
ExcititorIngestResumeRequest request,
IVexIngestOrchestrator orchestrator,
TimeProvider timeProvider,
CancellationToken cancellationToken)
internal static async Task<IResult> HandleResumeAsync(
HttpContext httpContext,
ExcititorIngestResumeRequest request,
IVexIngestOrchestrator orchestrator,
TimeProvider timeProvider,
CancellationToken cancellationToken)
{
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
if (scopeResult is not null)
{
return scopeResult;
}
_ = timeProvider;
var providerIds = NormalizeProviders(request.Providers);
var options = new IngestResumeOptions(providerIds, request.Checkpoint);
var summary = await orchestrator.ResumeAsync(options, cancellationToken).ConfigureAwait(false);
var message = $"Resume run completed for {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed.";
return TypedResults.Ok<object>(new
{
message,
runId = summary.RunId,
startedAt = summary.StartedAt,
completedAt = summary.CompletedAt,
durationMs = summary.Duration.TotalMilliseconds,
providers = summary.Providers.Select(static provider => new
{
providerId = provider.ProviderId,
status = provider.Status,
documents = provider.Documents,
claims = provider.Claims,
startedAt = provider.StartedAt,
completedAt = provider.CompletedAt,
durationMs = provider.Duration.TotalMilliseconds,
since = provider.Since,
checkpoint = provider.Checkpoint,
error = provider.Error
})
});
}
internal static async Task<IResult> HandleReconcileAsync(
HttpContext httpContext,
ExcititorReconcileRequest request,
IVexIngestOrchestrator orchestrator,
TimeProvider timeProvider,
CancellationToken cancellationToken)
{
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
if (scopeResult is not null)
@@ -127,81 +171,40 @@ internal static class IngestEndpoints
return scopeResult;
}
var providerIds = NormalizeProviders(request.Providers);
var options = new IngestResumeOptions(providerIds, request.Checkpoint, timeProvider);
var summary = await orchestrator.ResumeAsync(options, cancellationToken).ConfigureAwait(false);
var message = $"Resume run completed for {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed.";
return Results.Ok(new
{
message,
runId = summary.RunId,
startedAt = summary.StartedAt,
if (!TryParseTimeSpan(request.MaxAge, out var maxAge, out var error))
{
return TypedResults.BadRequest<object>(new { message = error });
}
_ = timeProvider;
var providerIds = NormalizeProviders(request.Providers);
var options = new ReconcileOptions(providerIds, maxAge);
var summary = await orchestrator.ReconcileAsync(options, cancellationToken).ConfigureAwait(false);
var message = $"Reconcile completed for {summary.ProviderCount} provider(s); {summary.ReconciledCount} reconciled, {summary.SkippedCount} skipped, {summary.FailureCount} failed.";
return TypedResults.Ok<object>(new
{
message,
runId = summary.RunId,
startedAt = summary.StartedAt,
completedAt = summary.CompletedAt,
durationMs = summary.Duration.TotalMilliseconds,
providers = summary.Providers.Select(static provider => new
{
provider.providerId,
provider.status,
provider.documents,
provider.claims,
provider.startedAt,
provider.completedAt,
provider.durationMs,
provider.since,
provider.checkpoint,
provider.error
})
});
}
durationMs = summary.Duration.TotalMilliseconds,
providers = summary.Providers.Select(static provider => new
{
providerId = provider.ProviderId,
status = provider.Status,
action = provider.Action,
lastUpdated = provider.LastUpdated,
threshold = provider.Threshold,
documents = provider.Documents,
claims = provider.Claims,
error = provider.Error
})
});
}
private static async Task<IResult> HandleReconcileAsync(
HttpContext httpContext,
ExcititorReconcileRequest request,
IVexIngestOrchestrator orchestrator,
TimeProvider timeProvider,
CancellationToken cancellationToken)
{
var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope);
if (scopeResult is not null)
{
return scopeResult;
}
if (!TryParseTimeSpan(request.MaxAge, out var maxAge, out var error))
{
return Results.BadRequest(new { message = error });
}
var providerIds = NormalizeProviders(request.Providers);
var options = new ReconcileOptions(providerIds, maxAge, timeProvider);
var summary = await orchestrator.ReconcileAsync(options, cancellationToken).ConfigureAwait(false);
var message = $"Reconcile completed for {summary.ProviderCount} provider(s); {summary.ReconciledCount} reconciled, {summary.SkippedCount} skipped, {summary.FailureCount} failed.";
return Results.Ok(new
{
message,
runId = summary.RunId,
startedAt = summary.StartedAt,
completedAt = summary.CompletedAt,
durationMs = summary.Duration.TotalMilliseconds,
providers = summary.Providers.Select(static provider => new
{
provider.providerId,
provider.status,
provider.action,
provider.lastUpdated,
provider.threshold,
provider.documents,
provider.claims,
provider.error
})
});
}
private static ImmutableArray<string> NormalizeProviders(IReadOnlyCollection<string>? providers)
internal static ImmutableArray<string> NormalizeProviders(IReadOnlyCollection<string>? providers)
{
if (providers is null || providers.Count == 0)
{
@@ -222,7 +225,7 @@ internal static class IngestEndpoints
return set.ToImmutableArray();
}
private static bool TryParseDateTimeOffset(string? value, out DateTimeOffset? result, out string? error)
internal static bool TryParseDateTimeOffset(string? value, out DateTimeOffset? result, out string? error)
{
result = null;
error = null;
@@ -246,7 +249,7 @@ internal static class IngestEndpoints
return false;
}
private static bool TryParseTimeSpan(string? value, out TimeSpan? result, out string? error)
internal static bool TryParseTimeSpan(string? value, out TimeSpan? result, out string? error)
{
result = null;
error = null;
@@ -266,19 +269,19 @@ internal static class IngestEndpoints
return false;
}
private sealed record ExcititorInitRequest(IReadOnlyList<string>? Providers, bool? Resume);
private sealed record ExcititorIngestRunRequest(
IReadOnlyList<string>? Providers,
string? Since,
string? Window,
bool? Force);
private sealed record ExcititorIngestResumeRequest(
IReadOnlyList<string>? Providers,
string? Checkpoint);
private sealed record ExcititorReconcileRequest(
IReadOnlyList<string>? Providers,
string? MaxAge);
}
internal sealed record ExcititorInitRequest(IReadOnlyList<string>? Providers, bool? Resume);
internal sealed record ExcititorIngestRunRequest(
IReadOnlyList<string>? Providers,
string? Since,
string? Window,
bool? Force);
internal sealed record ExcititorIngestResumeRequest(
IReadOnlyList<string>? Providers,
string? Checkpoint);
internal sealed record ExcititorReconcileRequest(
IReadOnlyList<string>? Providers,
string? MaxAge);
}

View File

@@ -15,16 +15,18 @@ using StellaOps.Excititor.Attestation.Dsse;
using StellaOps.Excititor.Attestation.Signing;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Policy;
using StellaOps.Excititor.Storage.Mongo;
internal static class ResolveEndpoint
{
private const int MaxSubjectPairs = 256;
public static void MapResolveEndpoint(WebApplication app)
{
app.MapPost("/excititor/resolve", HandleResolveAsync);
}
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.WebService.Services;
internal static class ResolveEndpoint
{
private const int MaxSubjectPairs = 256;
private const string ReadScope = "vex.read";
public static void MapResolveEndpoint(WebApplication app)
{
app.MapPost("/excititor/resolve", HandleResolveAsync);
}
private static async Task<IResult> HandleResolveAsync(
VexResolveRequest request,
@@ -38,13 +40,19 @@ internal static class ResolveEndpoint
IVexAttestationClient? attestationClient,
IVexSigner? signer,
CancellationToken cancellationToken)
{
if (request is null)
{
return Results.BadRequest("Request payload is required.");
}
var logger = loggerFactory.CreateLogger("ResolveEndpoint");
{
var scopeResult = ScopeAuthorization.RequireScope(httpContext, ReadScope);
if (scopeResult is not null)
{
return scopeResult;
}
if (request is null)
{
return Results.BadRequest("Request payload is required.");
}
var logger = loggerFactory.CreateLogger("ResolveEndpoint");
var productKeys = NormalizeValues(request.ProductKeys, request.Purls);
var vulnerabilityIds = NormalizeValues(request.VulnerabilityIds);

View File

@@ -0,0 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Excititor.WebService.Tests")]

View File

@@ -94,12 +94,12 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
await EnsureProviderRegistrationAsync(handle.Descriptor, session, cancellationToken).ConfigureAwait(false);
stopwatch.Stop();
results.Add(new InitProviderResult(
handle.Descriptor.Id,
handle.Descriptor.DisplayName,
"succeeded",
stopwatch.Elapsed,
error: null));
results.Add(new InitProviderResult(
handle.Descriptor.Id,
handle.Descriptor.DisplayName,
"succeeded",
stopwatch.Elapsed,
Error: null));
_logger.LogInformation("Excititor init validated provider {ProviderId} in {Duration}ms.", handle.Descriptor.Id, stopwatch.Elapsed.TotalMilliseconds);
}
@@ -223,15 +223,15 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
}
else
{
results.Add(new ReconcileProviderResult(
handle.Descriptor.Id,
"succeeded",
"skipped",
lastUpdated,
threshold,
documents: 0,
claims: 0,
error: null));
results.Add(new ReconcileProviderResult(
handle.Descriptor.Id,
"succeeded",
"skipped",
lastUpdated,
threshold,
Documents: 0,
Claims: 0,
Error: null));
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
@@ -299,19 +299,23 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
await ValidateConnectorAsync(handle, cancellationToken).ConfigureAwait(false);
await EnsureProviderRegistrationAsync(handle.Descriptor, session, cancellationToken).ConfigureAwait(false);
if (force)
{
var resetState = new VexConnectorState(providerId, null, ImmutableArray<string>.Empty);
await _stateRepository.SaveAsync(resetState, cancellationToken, session).ConfigureAwait(false);
}
var context = new VexConnectorContext(
since,
VexConnectorSettings.Empty,
_rawStore,
_signatureVerifier,
_normalizerRouter,
_serviceProvider);
if (force)
{
var resetState = new VexConnectorState(providerId, null, ImmutableArray<string>.Empty);
await _stateRepository.SaveAsync(resetState, cancellationToken, session).ConfigureAwait(false);
}
var stateBeforeRun = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false);
var resumeTokens = stateBeforeRun?.ResumeTokens ?? ImmutableDictionary<string, string>.Empty;
var context = new VexConnectorContext(
since,
VexConnectorSettings.Empty,
_rawStore,
_signatureVerifier,
_normalizerRouter,
_serviceProvider,
resumeTokens);
var documents = 0;
var claims = 0;
@@ -332,25 +336,25 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
stopwatch.Stop();
var completedAt = _timeProvider.GetUtcNow();
var state = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false);
var checkpoint = state?.DocumentDigests.IsDefaultOrEmpty == false
? state.DocumentDigests[^1]
: lastDigest;
var result = new ProviderRunResult(
providerId,
"succeeded",
var stateAfterRun = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false);
var checkpoint = stateAfterRun?.DocumentDigests.IsDefaultOrEmpty == false
? stateAfterRun.DocumentDigests[^1]
: lastDigest;
var result = new ProviderRunResult(
providerId,
"succeeded",
documents,
claims,
startedAt,
completedAt,
stopwatch.Elapsed,
lastDigest,
state?.LastUpdated,
checkpoint,
null,
since);
lastDigest,
stateAfterRun?.LastUpdated,
checkpoint,
null,
since);
_logger.LogInformation(
"Excititor ingest provider {ProviderId} completed: documents={Documents} claims={Claims} since={Since} duration={Duration}ms",

View File

@@ -3,7 +3,7 @@ If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|EXCITITOR-WEB-01-001 Minimal API bootstrap & DI|Team Excititor WebService|EXCITITOR-CORE-01-003, EXCITITOR-STORAGE-01-003|**DONE (2025-10-17)** Minimal API host composes storage/export/attestation/artifact stores, binds Mongo/attestation options, and exposes `/excititor/status` + health endpoints with regression coverage in `StatusEndpointTests`.|
|EXCITITOR-WEB-01-002 Ingest & reconcile endpoints|Team Excititor WebService|EXCITITOR-WEB-01-001|**DOING (2025-10-19)** Prereqs EXCITITOR-WEB-01-001, EXCITITOR-EXPORT-01-001, and EXCITITOR-ATTEST-01-001 verified DONE; drafting `/excititor/init`, `/excititor/ingest/run`, `/excititor/ingest/resume`, `/excititor/reconcile` with scope enforcement & structured telemetry plan.|
|EXCITITOR-WEB-01-002 Ingest & reconcile endpoints|Team Excititor WebService|EXCITITOR-WEB-01-001|**DONE (2025-10-20)** `/excititor/init`, `/excititor/ingest/run`, `/excititor/ingest/resume`, `/excititor/reconcile` enforce `vex.admin`, normalize provider lists, and return deterministic summaries; covered via unit tests (`dotnet test src/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj --filter FullyQualifiedName~IngestEndpointsTests`).|
|EXCITITOR-WEB-01-003 Export & verify endpoints|Team Excititor WebService|EXCITITOR-WEB-01-001, EXCITITOR-EXPORT-01-001, EXCITITOR-ATTEST-01-001|**DOING (2025-10-19)** Prereqs confirmed (EXCITITOR-WEB-01-001, EXCITITOR-EXPORT-01-001, EXCITITOR-ATTEST-01-001); preparing `/excititor/export*` surfaces and `/excititor/verify` with artifact/attestation metadata caching strategy.|
|EXCITITOR-WEB-01-004 Resolve API & signed responses|Team Excititor WebService|EXCITITOR-WEB-01-001, EXCITITOR-ATTEST-01-002|**DOING (2025-10-19)** Prereqs EXCITITOR-WEB-01-001, EXCITITOR-ATTEST-01-001, and EXCITITOR-ATTEST-01-002 verified DONE; planning `/excititor/resolve` signed response flow with consensus envelope + attestation metadata wiring.|
|EXCITITOR-WEB-01-004 Resolve API & signed responses|Team Excititor WebService|EXCITITOR-WEB-01-001, EXCITITOR-ATTEST-01-002|**DONE (2025-10-20)** Added `vex.read` scope enforcement, signed consensus/attestation envelopes, docs updates, and expanded tests (auth, unauthorized/forbidden). Mirror/ingest DTO casing fixed to restore builds.|
|EXCITITOR-WEB-01-005 Mirror distribution endpoints|Team Excititor WebService|EXCITITOR-EXPORT-01-007, DEVOPS-MIRROR-08-001|**DONE (2025-10-19)** `/excititor/mirror` surfaces domain listings, indices, metadata, and downloads with quota/auth checks; tests cover Happy-path listing/download (`dotnet test src/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj`).|