feat: Implement ScannerSurfaceSecretConfigurator for web service options
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Added ScannerSurfaceSecretConfigurator to configure ScannerWebServiceOptions using surface secrets.
- Integrated ISurfaceSecretProvider to fetch and apply secrets for artifact store configuration.
- Enhanced logging for secret retrieval and application processes.

feat: Implement ScannerStorageSurfaceSecretConfigurator for worker options

- Introduced ScannerStorageSurfaceSecretConfigurator to configure ScannerStorageOptions with surface secrets.
- Utilized ISurfaceSecretProvider to retrieve and apply secrets for object store settings.
- Improved logging for secret handling and configuration.

feat: Create SurfaceManifestPublisher for publishing surface manifests

- Developed SurfaceManifestPublisher to handle the creation and storage of surface manifests.
- Implemented methods for serializing manifest documents and storing payloads in the object store.
- Added dual write functionality for mirror storage of manifests.

feat: Add SurfaceManifestStageExecutor for processing scan stages

- Created SurfaceManifestStageExecutor to execute the manifest publishing stage in scan jobs.
- Integrated with SurfaceManifestPublisher to publish manifests based on collected payloads.
- Enhanced logging for job processing and manifest storage.

feat: Define SurfaceManifest models for manifest structure

- Established SurfaceManifestDocument, SurfaceManifestSource, SurfaceManifestArtifact, and SurfaceManifestStorage records.
- Implemented serialization attributes for JSON handling of manifest models.

feat: Implement CasAccessSecret and SurfaceSecretParser for secret handling

- Created CasAccessSecret record to represent surface access secrets.
- Developed SurfaceSecretParser to parse and validate surface secrets from JSON payloads.

test: Add unit tests for CasAccessSecretParser

- Implemented tests for parsing CasAccessSecret from JSON payloads and metadata fallbacks.
- Verified expected values and behavior for secret parsing logic.

test: Add unit tests for ScannerSurfaceSecretConfigurator

- Created tests for ScannerSurfaceSecretConfigurator to ensure correct application of surface secrets to web service options.
- Validated artifact store settings after configuration.

test: Add unit tests for ScannerStorageSurfaceSecretConfigurator

- Implemented tests for ScannerStorageSurfaceSecretConfigurator to verify correct application of surface secrets to storage options.
- Ensured accurate configuration of object store settings.
This commit is contained in:
master
2025-11-06 18:49:23 +02:00
parent e536492da9
commit 18f28168f0
33 changed files with 2066 additions and 621 deletions

View File

@@ -11,10 +11,10 @@ using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Hosting;
using System.Diagnostics;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using System.Diagnostics;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using MongoDB.Driver;
@@ -55,17 +55,17 @@ const string AdvisoryIngestPolicyName = "Concelier.Advisories.Ingest";
const string AdvisoryReadPolicyName = "Concelier.Advisories.Read";
const string AocVerifyPolicyName = "Concelier.Aoc.Verify";
const string TenantHeaderName = "X-Stella-Tenant";
builder.Configuration.AddStellaOpsDefaults(options =>
{
options.BasePath = builder.Environment.ContentRootPath;
options.EnvironmentPrefix = "CONCELIER_";
options.ConfigureBuilder = configurationBuilder =>
{
configurationBuilder.AddConcelierYaml(Path.Combine(builder.Environment.ContentRootPath, "../etc/concelier.yaml"));
};
});
builder.Configuration.AddStellaOpsDefaults(options =>
{
options.BasePath = builder.Environment.ContentRootPath;
options.EnvironmentPrefix = "CONCELIER_";
options.ConfigureBuilder = configurationBuilder =>
{
configurationBuilder.AddConcelierYaml(Path.Combine(builder.Environment.ContentRootPath, "../etc/concelier.yaml"));
};
});
var contentRootPath = builder.Environment.ContentRootPath;
var concelierOptions = builder.Configuration.BindOptions<ConcelierOptions>(postConfigure: (opts, _) =>
@@ -244,7 +244,7 @@ if (resolvedAuthority.Enabled && resolvedAuthority.AllowAnonymousFallback)
app.MapConcelierMirrorEndpoints(authorityConfigured, enforceAuthority);
app.MapGet("/.well-known/openapi", (OpenApiDiscoveryDocumentProvider provider, HttpContext context) =>
app.MapGet("/.well-known/openapi", ([FromServices] OpenApiDiscoveryDocumentProvider provider, HttpContext context) =>
{
var (payload, etag) = provider.GetDocument();
@@ -299,7 +299,7 @@ var observationsEndpoint = app.MapGet("/concelier/observations", async (
[FromQuery(Name = "cpe")] string[]? cpes,
[FromQuery(Name = "limit")] int? limit,
[FromQuery(Name = "cursor")] string? cursor,
IAdvisoryObservationQueryService queryService,
[FromServices] IAdvisoryObservationQueryService queryService,
CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
@@ -356,8 +356,8 @@ if (authorityConfigured)
var advisoryIngestEndpoint = app.MapPost("/ingest/advisory", async (
HttpContext context,
AdvisoryIngestRequest request,
IAdvisoryRawService rawService,
TimeProvider timeProvider,
[FromServices] IAdvisoryRawService rawService,
[FromServices] TimeProvider timeProvider,
CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
@@ -470,7 +470,7 @@ if (authorityConfigured)
var advisoryRawListEndpoint = app.MapGet("/advisories/raw", async (
HttpContext context,
IAdvisoryRawService rawService,
[FromServices] IAdvisoryRawService rawService,
CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
@@ -560,7 +560,7 @@ if (authorityConfigured)
var advisoryRawGetEndpoint = app.MapGet("/advisories/raw/{id}", async (
string id,
HttpContext context,
IAdvisoryRawService rawService,
[FromServices] IAdvisoryRawService rawService,
CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
@@ -604,7 +604,7 @@ if (authorityConfigured)
var advisoryRawProvenanceEndpoint = app.MapGet("/advisories/raw/{id}/provenance", async (
string id,
HttpContext context,
IAdvisoryRawService rawService,
[FromServices] IAdvisoryRawService rawService,
CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
@@ -650,8 +650,8 @@ if (authorityConfigured)
var aocVerifyEndpoint = app.MapPost("/aoc/verify", async (
HttpContext context,
AocVerifyRequest request,
IAdvisoryRawService rawService,
TimeProvider timeProvider,
[FromServices] IAdvisoryRawService rawService,
[FromServices] TimeProvider timeProvider,
CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
@@ -734,7 +734,7 @@ if (authorityConfigured)
app.MapGet("/concelier/advisories/{vulnerabilityKey}/replay", async (
string vulnerabilityKey,
DateTimeOffset? asOf,
IAdvisoryEventLog eventLog,
[FromServices] IAdvisoryEventLog eventLog,
CancellationToken cancellationToken) =>
{
if (string.IsNullOrWhiteSpace(vulnerabilityKey))
@@ -798,29 +798,29 @@ if (loggingEnabled)
};
});
}
app.UseExceptionHandler(errorApp =>
{
errorApp.Run(async context =>
{
context.Response.ContentType = "application/problem+json";
var feature = context.Features.Get<IExceptionHandlerFeature>();
var error = feature?.Error;
var extensions = new Dictionary<string, object?>(StringComparer.Ordinal)
{
["traceId"] = Activity.Current?.TraceId.ToString() ?? context.TraceIdentifier,
};
var problem = Results.Problem(
detail: error?.Message,
instance: context.Request.Path,
statusCode: StatusCodes.Status500InternalServerError,
title: "Unexpected server error",
type: ProblemTypes.JobFailure,
extensions: extensions);
await problem.ExecuteAsync(context);
var feature = context.Features.Get<IExceptionHandlerFeature>();
var error = feature?.Error;
var extensions = new Dictionary<string, object?>(StringComparer.Ordinal)
{
["traceId"] = Activity.Current?.TraceId.ToString() ?? context.TraceIdentifier,
};
var problem = Results.Problem(
detail: error?.Message,
instance: context.Request.Path,
statusCode: StatusCodes.Status500InternalServerError,
title: "Unexpected server error",
type: ProblemTypes.JobFailure,
extensions: extensions);
await problem.ExecuteAsync(context);
});
});
@@ -868,13 +868,13 @@ if (authorityConfigured)
app.UseAuthentication();
app.UseAuthorization();
}
IResult JsonResult<T>(T value, int? statusCode = null)
{
var payload = JsonSerializer.Serialize(value, jsonOptions);
return Results.Content(payload, "application/json", Encoding.UTF8, statusCode);
}
IResult JsonResult<T>(T value, int? statusCode = null)
{
var payload = JsonSerializer.Serialize(value, jsonOptions);
return Results.Content(payload, "application/json", Encoding.UTF8, statusCode);
}
IResult Problem(HttpContext context, string title, int statusCode, string type, string? detail = null, IDictionary<string, object?>? extensions = null)
{
var traceId = Activity.Current?.TraceId.ToString() ?? context.TraceIdentifier;
@@ -987,157 +987,157 @@ IResult MapAocGuardException(HttpContext context, ConcelierAocGuardException exc
var guardException = new AocGuardException(exception.Result);
return AocHttpResults.Problem(context, guardException);
}
static KeyValuePair<string, object?>[] BuildJobMetricTags(string jobKind, string trigger, string outcome)
=> new[]
{
new KeyValuePair<string, object?>("job.kind", jobKind),
new KeyValuePair<string, object?>("job.trigger", trigger),
new KeyValuePair<string, object?>("job.outcome", outcome),
};
void ApplyNoCache(HttpResponse response)
{
if (response is null)
{
return;
}
response.Headers.CacheControl = "no-store, no-cache, max-age=0, must-revalidate";
response.Headers.Pragma = "no-cache";
response.Headers["Expires"] = "0";
}
await InitializeMongoAsync(app);
app.MapGet("/health", (IOptions<ConcelierOptions> opts, ServiceStatus status, HttpContext context) =>
{
ApplyNoCache(context.Response);
var snapshot = status.CreateSnapshot();
var uptimeSeconds = Math.Max((snapshot.CapturedAt - snapshot.StartedAt).TotalSeconds, 0d);
var storage = new StorageBootstrapHealth(
Driver: opts.Value.Storage.Driver,
Completed: snapshot.BootstrapCompletedAt is not null,
CompletedAt: snapshot.BootstrapCompletedAt,
DurationMs: snapshot.BootstrapDuration?.TotalMilliseconds);
var telemetry = new TelemetryHealth(
Enabled: opts.Value.Telemetry.Enabled,
Tracing: opts.Value.Telemetry.EnableTracing,
Metrics: opts.Value.Telemetry.EnableMetrics,
Logging: opts.Value.Telemetry.EnableLogging);
var response = new HealthDocument(
Status: "healthy",
StartedAt: snapshot.StartedAt,
UptimeSeconds: uptimeSeconds,
Storage: storage,
Telemetry: telemetry);
return JsonResult(response);
});
app.MapGet("/ready", async (IMongoDatabase database, ServiceStatus status, HttpContext context, CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
var stopwatch = Stopwatch.StartNew();
try
{
await database.RunCommandAsync((Command<BsonDocument>)"{ ping: 1 }", cancellationToken: cancellationToken).ConfigureAwait(false);
stopwatch.Stop();
status.RecordMongoCheck(success: true, latency: stopwatch.Elapsed, error: null);
var snapshot = status.CreateSnapshot();
var uptimeSeconds = Math.Max((snapshot.CapturedAt - snapshot.StartedAt).TotalSeconds, 0d);
var mongo = new MongoReadyHealth(
Status: "ready",
LatencyMs: snapshot.LastMongoLatency?.TotalMilliseconds,
CheckedAt: snapshot.LastReadyCheckAt,
Error: null);
var response = new ReadyDocument(
Status: "ready",
StartedAt: snapshot.StartedAt,
UptimeSeconds: uptimeSeconds,
Mongo: mongo);
return JsonResult(response);
}
catch (Exception ex)
{
stopwatch.Stop();
status.RecordMongoCheck(success: false, latency: stopwatch.Elapsed, error: ex.Message);
var snapshot = status.CreateSnapshot();
var uptimeSeconds = Math.Max((snapshot.CapturedAt - snapshot.StartedAt).TotalSeconds, 0d);
var mongo = new MongoReadyHealth(
Status: "unready",
LatencyMs: snapshot.LastMongoLatency?.TotalMilliseconds,
CheckedAt: snapshot.LastReadyCheckAt,
Error: snapshot.LastMongoError ?? ex.Message);
var response = new ReadyDocument(
Status: "unready",
StartedAt: snapshot.StartedAt,
UptimeSeconds: uptimeSeconds,
Mongo: mongo);
var extensions = new Dictionary<string, object?>(StringComparer.Ordinal)
{
["mongoLatencyMs"] = snapshot.LastMongoLatency?.TotalMilliseconds,
["mongoError"] = snapshot.LastMongoError ?? ex.Message,
};
return Problem(context, "Mongo unavailable", StatusCodes.Status503ServiceUnavailable, ProblemTypes.ServiceUnavailable, snapshot.LastMongoError ?? ex.Message, extensions);
}
});
app.MapGet("/diagnostics/aliases/{seed}", async (string seed, AliasGraphResolver resolver, HttpContext context, CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
if (string.IsNullOrWhiteSpace(seed))
{
return Problem(context, "Seed advisory key is required.", StatusCodes.Status400BadRequest, ProblemTypes.Validation);
}
var component = await resolver.BuildComponentAsync(seed, cancellationToken).ConfigureAwait(false);
var aliases = component.AliasMap.ToDictionary(
static kvp => kvp.Key,
static kvp => kvp.Value
.Select(record => new
{
record.Scheme,
record.Value,
UpdatedAt = record.UpdatedAt
})
.ToArray());
var response = new
{
Seed = component.SeedAdvisoryKey,
Advisories = component.AdvisoryKeys,
Collisions = component.Collisions
.Select(collision => new
{
collision.Scheme,
collision.Value,
AdvisoryKeys = collision.AdvisoryKeys
})
.ToArray(),
Aliases = aliases
};
return JsonResult(response);
});
var jobsListEndpoint = app.MapGet("/jobs", async (string? kind, int? limit, IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) =>
static KeyValuePair<string, object?>[] BuildJobMetricTags(string jobKind, string trigger, string outcome)
=> new[]
{
new KeyValuePair<string, object?>("job.kind", jobKind),
new KeyValuePair<string, object?>("job.trigger", trigger),
new KeyValuePair<string, object?>("job.outcome", outcome),
};
void ApplyNoCache(HttpResponse response)
{
if (response is null)
{
return;
}
response.Headers.CacheControl = "no-store, no-cache, max-age=0, must-revalidate";
response.Headers.Pragma = "no-cache";
response.Headers["Expires"] = "0";
}
await InitializeMongoAsync(app);
app.MapGet("/health", ([FromServices] IOptions<ConcelierOptions> opts, [FromServices] ServiceStatus status, HttpContext context) =>
{
ApplyNoCache(context.Response);
var snapshot = status.CreateSnapshot();
var uptimeSeconds = Math.Max((snapshot.CapturedAt - snapshot.StartedAt).TotalSeconds, 0d);
var storage = new StorageBootstrapHealth(
Driver: opts.Value.Storage.Driver,
Completed: snapshot.BootstrapCompletedAt is not null,
CompletedAt: snapshot.BootstrapCompletedAt,
DurationMs: snapshot.BootstrapDuration?.TotalMilliseconds);
var telemetry = new TelemetryHealth(
Enabled: opts.Value.Telemetry.Enabled,
Tracing: opts.Value.Telemetry.EnableTracing,
Metrics: opts.Value.Telemetry.EnableMetrics,
Logging: opts.Value.Telemetry.EnableLogging);
var response = new HealthDocument(
Status: "healthy",
StartedAt: snapshot.StartedAt,
UptimeSeconds: uptimeSeconds,
Storage: storage,
Telemetry: telemetry);
return JsonResult(response);
});
app.MapGet("/ready", async ([FromServices] IMongoDatabase database, [FromServices] ServiceStatus status, HttpContext context, CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
var stopwatch = Stopwatch.StartNew();
try
{
await database.RunCommandAsync((Command<BsonDocument>)"{ ping: 1 }", cancellationToken: cancellationToken).ConfigureAwait(false);
stopwatch.Stop();
status.RecordMongoCheck(success: true, latency: stopwatch.Elapsed, error: null);
var snapshot = status.CreateSnapshot();
var uptimeSeconds = Math.Max((snapshot.CapturedAt - snapshot.StartedAt).TotalSeconds, 0d);
var mongo = new MongoReadyHealth(
Status: "ready",
LatencyMs: snapshot.LastMongoLatency?.TotalMilliseconds,
CheckedAt: snapshot.LastReadyCheckAt,
Error: null);
var response = new ReadyDocument(
Status: "ready",
StartedAt: snapshot.StartedAt,
UptimeSeconds: uptimeSeconds,
Mongo: mongo);
return JsonResult(response);
}
catch (Exception ex)
{
stopwatch.Stop();
status.RecordMongoCheck(success: false, latency: stopwatch.Elapsed, error: ex.Message);
var snapshot = status.CreateSnapshot();
var uptimeSeconds = Math.Max((snapshot.CapturedAt - snapshot.StartedAt).TotalSeconds, 0d);
var mongo = new MongoReadyHealth(
Status: "unready",
LatencyMs: snapshot.LastMongoLatency?.TotalMilliseconds,
CheckedAt: snapshot.LastReadyCheckAt,
Error: snapshot.LastMongoError ?? ex.Message);
var response = new ReadyDocument(
Status: "unready",
StartedAt: snapshot.StartedAt,
UptimeSeconds: uptimeSeconds,
Mongo: mongo);
var extensions = new Dictionary<string, object?>(StringComparer.Ordinal)
{
["mongoLatencyMs"] = snapshot.LastMongoLatency?.TotalMilliseconds,
["mongoError"] = snapshot.LastMongoError ?? ex.Message,
};
return Problem(context, "Mongo unavailable", StatusCodes.Status503ServiceUnavailable, ProblemTypes.ServiceUnavailable, snapshot.LastMongoError ?? ex.Message, extensions);
}
});
app.MapGet("/diagnostics/aliases/{seed}", async (string seed, [FromServices] AliasGraphResolver resolver, HttpContext context, CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
if (string.IsNullOrWhiteSpace(seed))
{
return Problem(context, "Seed advisory key is required.", StatusCodes.Status400BadRequest, ProblemTypes.Validation);
}
var component = await resolver.BuildComponentAsync(seed, cancellationToken).ConfigureAwait(false);
var aliases = component.AliasMap.ToDictionary(
static kvp => kvp.Key,
static kvp => kvp.Value
.Select(record => new
{
record.Scheme,
record.Value,
UpdatedAt = record.UpdatedAt
})
.ToArray());
var response = new
{
Seed = component.SeedAdvisoryKey,
Advisories = component.AdvisoryKeys,
Collisions = component.Collisions
.Select(collision => new
{
collision.Scheme,
collision.Value,
AdvisoryKeys = collision.AdvisoryKeys
})
.ToArray(),
Aliases = aliases
};
return JsonResult(response);
});
var jobsListEndpoint = app.MapGet("/jobs", async (string? kind, int? limit, [FromServices] IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
@@ -1151,7 +1151,7 @@ if (enforceAuthority)
jobsListEndpoint.RequireAuthorization(JobsPolicyName);
}
var jobByIdEndpoint = app.MapGet("/jobs/{runId:guid}", async (Guid runId, IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) =>
var jobByIdEndpoint = app.MapGet("/jobs/{runId:guid}", async (Guid runId, [FromServices] IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
@@ -1168,25 +1168,25 @@ if (enforceAuthority)
jobByIdEndpoint.RequireAuthorization(JobsPolicyName);
}
var jobDefinitionsEndpoint = app.MapGet("/jobs/definitions", async (IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) =>
var jobDefinitionsEndpoint = app.MapGet("/jobs/definitions", async ([FromServices] IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
var definitions = await coordinator.GetDefinitionsAsync(cancellationToken).ConfigureAwait(false);
if (definitions.Count == 0)
{
return JsonResult(Array.Empty<JobDefinitionResponse>());
}
var definitionKinds = definitions.Select(static definition => definition.Kind).ToArray();
var lastRuns = await coordinator.GetLastRunsAsync(definitionKinds, cancellationToken).ConfigureAwait(false);
var responses = new List<JobDefinitionResponse>(definitions.Count);
foreach (var definition in definitions)
{
lastRuns.TryGetValue(definition.Kind, out var lastRun);
responses.Add(JobDefinitionResponse.FromDefinition(definition, lastRun));
}
ApplyNoCache(context.Response);
var definitions = await coordinator.GetDefinitionsAsync(cancellationToken).ConfigureAwait(false);
if (definitions.Count == 0)
{
return JsonResult(Array.Empty<JobDefinitionResponse>());
}
var definitionKinds = definitions.Select(static definition => definition.Kind).ToArray();
var lastRuns = await coordinator.GetLastRunsAsync(definitionKinds, cancellationToken).ConfigureAwait(false);
var responses = new List<JobDefinitionResponse>(definitions.Count);
foreach (var definition in definitions)
{
lastRuns.TryGetValue(definition.Kind, out var lastRun);
responses.Add(JobDefinitionResponse.FromDefinition(definition, lastRun));
}
return JsonResult(responses);
}).AddEndpointFilter<JobAuthorizationAuditFilter>();
@@ -1195,20 +1195,20 @@ if (enforceAuthority)
jobDefinitionsEndpoint.RequireAuthorization(JobsPolicyName);
}
var jobDefinitionEndpoint = app.MapGet("/jobs/definitions/{kind}", async (string kind, IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) =>
var jobDefinitionEndpoint = app.MapGet("/jobs/definitions/{kind}", async (string kind, [FromServices] IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
var definition = (await coordinator.GetDefinitionsAsync(cancellationToken).ConfigureAwait(false))
.FirstOrDefault(d => string.Equals(d.Kind, kind, StringComparison.Ordinal));
if (definition is null)
{
return Problem(context, "Job definition not found", StatusCodes.Status404NotFound, ProblemTypes.NotFound, $"Job kind '{kind}' is not registered.");
}
var lastRuns = await coordinator.GetLastRunsAsync(new[] { definition.Kind }, cancellationToken).ConfigureAwait(false);
lastRuns.TryGetValue(definition.Kind, out var lastRun);
ApplyNoCache(context.Response);
var definition = (await coordinator.GetDefinitionsAsync(cancellationToken).ConfigureAwait(false))
.FirstOrDefault(d => string.Equals(d.Kind, kind, StringComparison.Ordinal));
if (definition is null)
{
return Problem(context, "Job definition not found", StatusCodes.Status404NotFound, ProblemTypes.NotFound, $"Job kind '{kind}' is not registered.");
}
var lastRuns = await coordinator.GetLastRunsAsync(new[] { definition.Kind }, cancellationToken).ConfigureAwait(false);
lastRuns.TryGetValue(definition.Kind, out var lastRun);
var response = JobDefinitionResponse.FromDefinition(definition, lastRun);
return JsonResult(response);
@@ -1218,18 +1218,18 @@ if (enforceAuthority)
jobDefinitionEndpoint.RequireAuthorization(JobsPolicyName);
}
var jobDefinitionRunsEndpoint = app.MapGet("/jobs/definitions/{kind}/runs", async (string kind, int? limit, IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) =>
var jobDefinitionRunsEndpoint = app.MapGet("/jobs/definitions/{kind}/runs", async (string kind, int? limit, [FromServices] IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
var definition = (await coordinator.GetDefinitionsAsync(cancellationToken).ConfigureAwait(false))
.FirstOrDefault(d => string.Equals(d.Kind, kind, StringComparison.Ordinal));
if (definition is null)
{
return Problem(context, "Job definition not found", StatusCodes.Status404NotFound, ProblemTypes.NotFound, $"Job kind '{kind}' is not registered.");
}
ApplyNoCache(context.Response);
var definition = (await coordinator.GetDefinitionsAsync(cancellationToken).ConfigureAwait(false))
.FirstOrDefault(d => string.Equals(d.Kind, kind, StringComparison.Ordinal));
if (definition is null)
{
return Problem(context, "Job definition not found", StatusCodes.Status404NotFound, ProblemTypes.NotFound, $"Job kind '{kind}' is not registered.");
}
var take = Math.Clamp(limit.GetValueOrDefault(20), 1, 200);
var runs = await coordinator.GetRecentRunsAsync(kind, take, cancellationToken).ConfigureAwait(false);
var payload = runs.Select(JobRunResponse.FromSnapshot).ToArray();
@@ -1240,9 +1240,9 @@ if (enforceAuthority)
jobDefinitionRunsEndpoint.RequireAuthorization(JobsPolicyName);
}
var activeJobsEndpoint = app.MapGet("/jobs/active", async (IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) =>
var activeJobsEndpoint = app.MapGet("/jobs/active", async ([FromServices] IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
ApplyNoCache(context.Response);
var runs = await coordinator.GetActiveRunsAsync(cancellationToken).ConfigureAwait(false);
var payload = runs.Select(JobRunResponse.FromSnapshot).ToArray();
@@ -1253,22 +1253,22 @@ if (enforceAuthority)
activeJobsEndpoint.RequireAuthorization(JobsPolicyName);
}
var triggerJobEndpoint = app.MapPost("/jobs/{*jobKind}", async (string jobKind, JobTriggerRequest request, IJobCoordinator coordinator, HttpContext context) =>
var triggerJobEndpoint = app.MapPost("/jobs/{*jobKind}", async (string jobKind, JobTriggerRequest request, [FromServices] IJobCoordinator coordinator, HttpContext context) =>
{
ApplyNoCache(context.Response);
request ??= new JobTriggerRequest();
request.Parameters ??= new Dictionary<string, object?>(StringComparer.Ordinal);
var trigger = string.IsNullOrWhiteSpace(request.Trigger) ? "api" : request.Trigger;
var lifetime = context.RequestServices.GetRequiredService<IHostApplicationLifetime>();
var result = await coordinator.TriggerAsync(jobKind, request.Parameters, trigger, lifetime.ApplicationStopping).ConfigureAwait(false);
var outcome = result.Outcome;
var tags = BuildJobMetricTags(jobKind, trigger, outcome.ToString().ToLowerInvariant());
switch (outcome)
{
ApplyNoCache(context.Response);
request ??= new JobTriggerRequest();
request.Parameters ??= new Dictionary<string, object?>(StringComparer.Ordinal);
var trigger = string.IsNullOrWhiteSpace(request.Trigger) ? "api" : request.Trigger;
var lifetime = context.RequestServices.GetRequiredService<IHostApplicationLifetime>();
var result = await coordinator.TriggerAsync(jobKind, request.Parameters, trigger, lifetime.ApplicationStopping).ConfigureAwait(false);
var outcome = result.Outcome;
var tags = BuildJobMetricTags(jobKind, trigger, outcome.ToString().ToLowerInvariant());
switch (outcome)
{
case JobTriggerOutcome.Accepted:
JobMetrics.TriggerCounter.Add(1, tags);
if (result.Run is null)
@@ -1279,54 +1279,54 @@ var triggerJobEndpoint = app.MapPost("/jobs/{*jobKind}", async (string jobKind,
var acceptedRun = JobRunResponse.FromSnapshot(result.Run);
context.Response.Headers.Location = $"/jobs/{acceptedRun.RunId}";
return JsonResult(acceptedRun, StatusCodes.Status202Accepted);
case JobTriggerOutcome.NotFound:
JobMetrics.TriggerConflictCounter.Add(1, tags);
return Problem(context, "Job not found", StatusCodes.Status404NotFound, ProblemTypes.NotFound, result.ErrorMessage ?? $"Job '{jobKind}' is not registered.");
case JobTriggerOutcome.Disabled:
JobMetrics.TriggerConflictCounter.Add(1, tags);
return Problem(context, "Job disabled", StatusCodes.Status423Locked, ProblemTypes.Locked, result.ErrorMessage ?? $"Job '{jobKind}' is disabled.");
case JobTriggerOutcome.AlreadyRunning:
JobMetrics.TriggerConflictCounter.Add(1, tags);
return Problem(context, "Job already running", StatusCodes.Status409Conflict, ProblemTypes.Conflict, result.ErrorMessage ?? $"Job '{jobKind}' already has an active run.");
case JobTriggerOutcome.LeaseRejected:
JobMetrics.TriggerConflictCounter.Add(1, tags);
return Problem(context, "Job lease rejected", StatusCodes.Status409Conflict, ProblemTypes.LeaseRejected, result.ErrorMessage ?? $"Job '{jobKind}' could not acquire a lease.");
case JobTriggerOutcome.InvalidParameters:
{
JobMetrics.TriggerConflictCounter.Add(1, tags);
var extensions = new Dictionary<string, object?>(StringComparer.Ordinal)
{
["parameters"] = request.Parameters,
};
return Problem(context, "Invalid job parameters", StatusCodes.Status400BadRequest, ProblemTypes.Validation, result.ErrorMessage, extensions);
}
case JobTriggerOutcome.Cancelled:
{
JobMetrics.TriggerConflictCounter.Add(1, tags);
var extensions = new Dictionary<string, object?>(StringComparer.Ordinal)
{
["run"] = result.Run is null ? null : JobRunResponse.FromSnapshot(result.Run),
};
return Problem(context, "Job cancelled", StatusCodes.Status409Conflict, ProblemTypes.Conflict, result.ErrorMessage ?? $"Job '{jobKind}' was cancelled before completion.", extensions);
}
case JobTriggerOutcome.Failed:
{
JobMetrics.TriggerFailureCounter.Add(1, tags);
var extensions = new Dictionary<string, object?>(StringComparer.Ordinal)
{
["run"] = result.Run is null ? null : JobRunResponse.FromSnapshot(result.Run),
};
return Problem(context, "Job execution failed", StatusCodes.Status500InternalServerError, ProblemTypes.JobFailure, result.ErrorMessage, extensions);
}
case JobTriggerOutcome.NotFound:
JobMetrics.TriggerConflictCounter.Add(1, tags);
return Problem(context, "Job not found", StatusCodes.Status404NotFound, ProblemTypes.NotFound, result.ErrorMessage ?? $"Job '{jobKind}' is not registered.");
case JobTriggerOutcome.Disabled:
JobMetrics.TriggerConflictCounter.Add(1, tags);
return Problem(context, "Job disabled", StatusCodes.Status423Locked, ProblemTypes.Locked, result.ErrorMessage ?? $"Job '{jobKind}' is disabled.");
case JobTriggerOutcome.AlreadyRunning:
JobMetrics.TriggerConflictCounter.Add(1, tags);
return Problem(context, "Job already running", StatusCodes.Status409Conflict, ProblemTypes.Conflict, result.ErrorMessage ?? $"Job '{jobKind}' already has an active run.");
case JobTriggerOutcome.LeaseRejected:
JobMetrics.TriggerConflictCounter.Add(1, tags);
return Problem(context, "Job lease rejected", StatusCodes.Status409Conflict, ProblemTypes.LeaseRejected, result.ErrorMessage ?? $"Job '{jobKind}' could not acquire a lease.");
case JobTriggerOutcome.InvalidParameters:
{
JobMetrics.TriggerConflictCounter.Add(1, tags);
var extensions = new Dictionary<string, object?>(StringComparer.Ordinal)
{
["parameters"] = request.Parameters,
};
return Problem(context, "Invalid job parameters", StatusCodes.Status400BadRequest, ProblemTypes.Validation, result.ErrorMessage, extensions);
}
case JobTriggerOutcome.Cancelled:
{
JobMetrics.TriggerConflictCounter.Add(1, tags);
var extensions = new Dictionary<string, object?>(StringComparer.Ordinal)
{
["run"] = result.Run is null ? null : JobRunResponse.FromSnapshot(result.Run),
};
return Problem(context, "Job cancelled", StatusCodes.Status409Conflict, ProblemTypes.Conflict, result.ErrorMessage ?? $"Job '{jobKind}' was cancelled before completion.", extensions);
}
case JobTriggerOutcome.Failed:
{
JobMetrics.TriggerFailureCounter.Add(1, tags);
var extensions = new Dictionary<string, object?>(StringComparer.Ordinal)
{
["run"] = result.Run is null ? null : JobRunResponse.FromSnapshot(result.Run),
};
return Problem(context, "Job execution failed", StatusCodes.Status500InternalServerError, ProblemTypes.JobFailure, result.ErrorMessage, extensions);
}
default:
JobMetrics.TriggerFailureCounter.Add(1, tags);
@@ -1337,61 +1337,61 @@ if (enforceAuthority)
{
triggerJobEndpoint.RequireAuthorization(JobsPolicyName);
}
await app.RunAsync();
static PluginHostOptions BuildPluginOptions(ConcelierOptions options, string contentRoot)
{
var pluginOptions = new PluginHostOptions
{
await app.RunAsync();
static PluginHostOptions BuildPluginOptions(ConcelierOptions options, string contentRoot)
{
var pluginOptions = new PluginHostOptions
{
BaseDirectory = options.Plugins.BaseDirectory ?? contentRoot,
PluginsDirectory = options.Plugins.Directory ?? Path.Combine(contentRoot, "StellaOps.Concelier.PluginBinaries"),
PrimaryPrefix = "StellaOps.Concelier",
EnsureDirectoryExists = true,
RecursiveSearch = false,
};
if (options.Plugins.SearchPatterns.Count == 0)
{
pluginOptions.SearchPatterns.Add("StellaOps.Concelier.Plugin.*.dll");
}
else
{
foreach (var pattern in options.Plugins.SearchPatterns)
{
if (!string.IsNullOrWhiteSpace(pattern))
{
pluginOptions.SearchPatterns.Add(pattern);
}
}
}
return pluginOptions;
}
static async Task InitializeMongoAsync(WebApplication app)
{
await using var scope = app.Services.CreateAsyncScope();
var bootstrapper = scope.ServiceProvider.GetRequiredService<MongoBootstrapper>();
var logger = scope.ServiceProvider.GetRequiredService<ILoggerFactory>().CreateLogger("MongoBootstrapper");
var status = scope.ServiceProvider.GetRequiredService<ServiceStatus>();
var stopwatch = Stopwatch.StartNew();
try
{
await bootstrapper.InitializeAsync(app.Lifetime.ApplicationStopping).ConfigureAwait(false);
stopwatch.Stop();
status.MarkBootstrapCompleted(stopwatch.Elapsed);
logger.LogInformation("Mongo bootstrap completed in {ElapsedMs} ms", stopwatch.Elapsed.TotalMilliseconds);
}
catch (Exception ex)
{
stopwatch.Stop();
status.RecordMongoCheck(success: false, latency: stopwatch.Elapsed, error: ex.Message);
logger.LogCritical(ex, "Mongo bootstrap failed after {ElapsedMs} ms", stopwatch.Elapsed.TotalMilliseconds);
throw;
}
}
public partial class Program;
EnsureDirectoryExists = true,
RecursiveSearch = false,
};
if (options.Plugins.SearchPatterns.Count == 0)
{
pluginOptions.SearchPatterns.Add("StellaOps.Concelier.Plugin.*.dll");
}
else
{
foreach (var pattern in options.Plugins.SearchPatterns)
{
if (!string.IsNullOrWhiteSpace(pattern))
{
pluginOptions.SearchPatterns.Add(pattern);
}
}
}
return pluginOptions;
}
static async Task InitializeMongoAsync(WebApplication app)
{
await using var scope = app.Services.CreateAsyncScope();
var bootstrapper = scope.ServiceProvider.GetRequiredService<MongoBootstrapper>();
var logger = scope.ServiceProvider.GetRequiredService<ILoggerFactory>().CreateLogger("MongoBootstrapper");
var status = scope.ServiceProvider.GetRequiredService<ServiceStatus>();
var stopwatch = Stopwatch.StartNew();
try
{
await bootstrapper.InitializeAsync(app.Lifetime.ApplicationStopping).ConfigureAwait(false);
stopwatch.Stop();
status.MarkBootstrapCompleted(stopwatch.Elapsed);
logger.LogInformation("Mongo bootstrap completed in {ElapsedMs} ms", stopwatch.Elapsed.TotalMilliseconds);
}
catch (Exception ex)
{
stopwatch.Stop();
status.RecordMongoCheck(success: false, latency: stopwatch.Elapsed, error: ex.Message);
logger.LogCritical(ex, "Mongo bootstrap failed after {ElapsedMs} ms", stopwatch.Elapsed.TotalMilliseconds);
throw;
}
}
public partial class Program;