Restructure solution layout by module
This commit is contained in:
34
src/Concelier/StellaOps.Concelier.WebService/AGENTS.md
Normal file
34
src/Concelier/StellaOps.Concelier.WebService/AGENTS.md
Normal file
@@ -0,0 +1,34 @@
|
||||
# AGENTS
|
||||
## Role
|
||||
Minimal API host wiring configuration, storage, plugin routines, and job endpoints. Operational surface for health, readiness, and job control.
|
||||
## Scope
|
||||
- Configuration: appsettings.json + etc/concelier.yaml (yaml path = ../etc/concelier.yaml); bind into ConcelierOptions with validation (Only Mongo supported).
|
||||
- Mongo: MongoUrl from options.Storage.Dsn; IMongoClient/IMongoDatabase singletons; default database name fallback (options -> URL -> "concelier").
|
||||
- Services: AddMongoStorage(); AddSourceHttpClients(); RegisterPluginRoutines(configuration, PluginHostOptions).
|
||||
- Bootstrap: MongoBootstrapper.InitializeAsync on startup.
|
||||
- Endpoints (configuration & job control only; root path intentionally unbound):
|
||||
- GET /health -> {status:"healthy"} after options validation binds.
|
||||
- GET /ready -> MongoDB ping; 503 on MongoException/Timeout.
|
||||
- GET /jobs?kind=&limit= -> recent runs.
|
||||
- GET /jobs/{id} -> run detail.
|
||||
- GET /jobs/definitions -> definitions with lastRun.
|
||||
- GET /jobs/definitions/{kind} -> definition + lastRun or 404.
|
||||
- GET /jobs/definitions/{kind}/runs?limit= -> recent runs or 404 if kind unknown.
|
||||
- GET /jobs/active -> currently running.
|
||||
- POST /jobs/{*jobKind} with {trigger?,parameters?} -> 202 Accepted (Location:/jobs/{runId}) | 404 | 409 | 423.
|
||||
- PluginHost defaults: BaseDirectory = solution root; PluginsDirectory = "StellaOps.Concelier.PluginBinaries"; SearchPatterns += "StellaOps.Concelier.Plugin.*.dll"; EnsureDirectoryExists = true.
|
||||
## Participants
|
||||
- Core job system; Storage.Mongo; Source.Common HTTP clients; Exporter and Connector plugin routines discover/register jobs.
|
||||
## Interfaces & contracts
|
||||
- Dependency injection boundary for all connectors/exporters; IOptions<ConcelierOptions> validated on start.
|
||||
- Cancellation: pass app.Lifetime.ApplicationStopping to bootstrapper.
|
||||
## In/Out of scope
|
||||
In: hosting, DI composition, REST surface, readiness checks.
|
||||
Out: business logic of jobs, HTML UI, authn/z (future).
|
||||
## Observability & security expectations
|
||||
- Log startup config (redact DSN credentials), plugin scan results (missing ordered plugins if any).
|
||||
- Structured responses with status codes; no stack traces in HTTP bodies; errors mapped cleanly.
|
||||
## Tests
|
||||
- Author and review coverage in `../StellaOps.Concelier.WebService.Tests`.
|
||||
- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`.
|
||||
- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios.
|
||||
@@ -0,0 +1,16 @@
|
||||
using System.Collections.Immutable;
|
||||
using StellaOps.Concelier.Models.Observations;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Contracts;
|
||||
|
||||
public sealed record AdvisoryObservationQueryResponse(
|
||||
ImmutableArray<AdvisoryObservation> Observations,
|
||||
AdvisoryObservationLinksetAggregateResponse Linkset,
|
||||
string? NextCursor,
|
||||
bool HasMore);
|
||||
|
||||
public sealed record AdvisoryObservationLinksetAggregateResponse(
|
||||
ImmutableArray<string> Aliases,
|
||||
ImmutableArray<string> Purls,
|
||||
ImmutableArray<string> Cpes,
|
||||
ImmutableArray<AdvisoryObservationReference> References);
|
||||
@@ -0,0 +1,127 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using StellaOps.Concelier.RawModels;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Contracts;
|
||||
|
||||
public sealed record AdvisoryIngestRequest(
|
||||
AdvisorySourceRequest Source,
|
||||
AdvisoryUpstreamRequest Upstream,
|
||||
AdvisoryContentRequest Content,
|
||||
AdvisoryIdentifiersRequest Identifiers,
|
||||
AdvisoryLinksetRequest? Linkset);
|
||||
|
||||
public sealed record AdvisorySourceRequest(
|
||||
[property: JsonPropertyName("vendor")] string Vendor,
|
||||
[property: JsonPropertyName("connector")] string Connector,
|
||||
[property: JsonPropertyName("version")] string Version,
|
||||
[property: JsonPropertyName("stream")] string? Stream);
|
||||
|
||||
public sealed record AdvisoryUpstreamRequest(
|
||||
[property: JsonPropertyName("upstreamId")] string UpstreamId,
|
||||
[property: JsonPropertyName("documentVersion")] string? DocumentVersion,
|
||||
[property: JsonPropertyName("retrievedAt")] DateTimeOffset? RetrievedAt,
|
||||
[property: JsonPropertyName("contentHash")] string ContentHash,
|
||||
[property: JsonPropertyName("signature")] AdvisorySignatureRequest Signature,
|
||||
[property: JsonPropertyName("provenance")] IDictionary<string, string>? Provenance);
|
||||
|
||||
public sealed record AdvisorySignatureRequest(
|
||||
[property: JsonPropertyName("present")] bool Present,
|
||||
[property: JsonPropertyName("format")] string? Format,
|
||||
[property: JsonPropertyName("keyId")] string? KeyId,
|
||||
[property: JsonPropertyName("sig")] string? Signature,
|
||||
[property: JsonPropertyName("certificate")] string? Certificate,
|
||||
[property: JsonPropertyName("digest")] string? Digest);
|
||||
|
||||
public sealed record AdvisoryContentRequest(
|
||||
[property: JsonPropertyName("format")] string Format,
|
||||
[property: JsonPropertyName("specVersion")] string? SpecVersion,
|
||||
[property: JsonPropertyName("raw")] JsonElement Raw,
|
||||
[property: JsonPropertyName("encoding")] string? Encoding);
|
||||
|
||||
public sealed record AdvisoryIdentifiersRequest(
|
||||
[property: JsonPropertyName("primary")] string Primary,
|
||||
[property: JsonPropertyName("aliases")] IReadOnlyList<string>? Aliases);
|
||||
|
||||
public sealed record AdvisoryLinksetRequest(
|
||||
[property: JsonPropertyName("aliases")] IReadOnlyList<string>? Aliases,
|
||||
[property: JsonPropertyName("purls")] IReadOnlyList<string>? PackageUrls,
|
||||
[property: JsonPropertyName("cpes")] IReadOnlyList<string>? Cpes,
|
||||
[property: JsonPropertyName("references")] IReadOnlyList<AdvisoryLinksetReferenceRequest>? References,
|
||||
[property: JsonPropertyName("reconciledFrom")] IReadOnlyList<string>? ReconciledFrom,
|
||||
[property: JsonPropertyName("notes")] IDictionary<string, string>? Notes);
|
||||
|
||||
public sealed record AdvisoryLinksetReferenceRequest(
|
||||
[property: JsonPropertyName("type")] string Type,
|
||||
[property: JsonPropertyName("url")] string Url,
|
||||
[property: JsonPropertyName("source")] string? Source);
|
||||
|
||||
public sealed record AdvisoryIngestResponse(
|
||||
[property: JsonPropertyName("id")] string Id,
|
||||
[property: JsonPropertyName("inserted")] bool Inserted,
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("contentHash")] string ContentHash,
|
||||
[property: JsonPropertyName("supersedes")] string? Supersedes,
|
||||
[property: JsonPropertyName("ingestedAt")] DateTimeOffset IngestedAt,
|
||||
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt);
|
||||
|
||||
public sealed record AdvisoryRawRecordResponse(
|
||||
[property: JsonPropertyName("id")] string Id,
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("ingestedAt")] DateTimeOffset IngestedAt,
|
||||
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt,
|
||||
[property: JsonPropertyName("document")] AdvisoryRawDocument Document);
|
||||
|
||||
public sealed record AdvisoryRawListResponse(
|
||||
[property: JsonPropertyName("records")] IReadOnlyList<AdvisoryRawRecordResponse> Records,
|
||||
[property: JsonPropertyName("nextCursor")] string? NextCursor,
|
||||
[property: JsonPropertyName("hasMore")] bool HasMore);
|
||||
|
||||
public sealed record AdvisoryRawProvenanceResponse(
|
||||
[property: JsonPropertyName("id")] string Id,
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("source")] RawSourceMetadata Source,
|
||||
[property: JsonPropertyName("upstream")] RawUpstreamMetadata Upstream,
|
||||
[property: JsonPropertyName("supersedes")] string? Supersedes,
|
||||
[property: JsonPropertyName("ingestedAt")] DateTimeOffset IngestedAt,
|
||||
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt);
|
||||
|
||||
public sealed record AocVerifyRequest(
|
||||
[property: JsonPropertyName("since")] DateTimeOffset? Since,
|
||||
[property: JsonPropertyName("until")] DateTimeOffset? Until,
|
||||
[property: JsonPropertyName("limit")] int? Limit,
|
||||
[property: JsonPropertyName("sources")] IReadOnlyList<string>? Sources,
|
||||
[property: JsonPropertyName("codes")] IReadOnlyList<string>? Codes);
|
||||
|
||||
public sealed record AocVerifyResponse(
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("window")] AocVerifyWindow Window,
|
||||
[property: JsonPropertyName("checked")] AocVerifyChecked Checked,
|
||||
[property: JsonPropertyName("violations")] IReadOnlyList<AocVerifyViolation> Violations,
|
||||
[property: JsonPropertyName("metrics")] AocVerifyMetrics Metrics,
|
||||
[property: JsonPropertyName("truncated")] bool Truncated);
|
||||
|
||||
public sealed record AocVerifyWindow(
|
||||
[property: JsonPropertyName("from")] DateTimeOffset From,
|
||||
[property: JsonPropertyName("to")] DateTimeOffset To);
|
||||
|
||||
public sealed record AocVerifyChecked(
|
||||
[property: JsonPropertyName("advisories")] int Advisories,
|
||||
[property: JsonPropertyName("vex")] int Vex);
|
||||
|
||||
public sealed record AocVerifyMetrics(
|
||||
[property: JsonPropertyName("ingestion_write_total")] int IngestionWriteTotal,
|
||||
[property: JsonPropertyName("aoc_violation_total")] int AocViolationTotal);
|
||||
|
||||
public sealed record AocVerifyViolation(
|
||||
[property: JsonPropertyName("code")] string Code,
|
||||
[property: JsonPropertyName("count")] int Count,
|
||||
[property: JsonPropertyName("examples")] IReadOnlyList<AocVerifyViolationExample> Examples);
|
||||
|
||||
public sealed record AocVerifyViolationExample(
|
||||
[property: JsonPropertyName("source")] string Source,
|
||||
[property: JsonPropertyName("documentId")] string DocumentId,
|
||||
[property: JsonPropertyName("contentHash")] string ContentHash,
|
||||
[property: JsonPropertyName("path")] string Path);
|
||||
@@ -0,0 +1,32 @@
|
||||
namespace StellaOps.Concelier.WebService.Diagnostics;
|
||||
|
||||
internal sealed record StorageBootstrapHealth(
|
||||
string Driver,
|
||||
bool Completed,
|
||||
DateTimeOffset? CompletedAt,
|
||||
double? DurationMs);
|
||||
|
||||
internal sealed record TelemetryHealth(
|
||||
bool Enabled,
|
||||
bool Tracing,
|
||||
bool Metrics,
|
||||
bool Logging);
|
||||
|
||||
internal sealed record HealthDocument(
|
||||
string Status,
|
||||
DateTimeOffset StartedAt,
|
||||
double UptimeSeconds,
|
||||
StorageBootstrapHealth Storage,
|
||||
TelemetryHealth Telemetry);
|
||||
|
||||
internal sealed record MongoReadyHealth(
|
||||
string Status,
|
||||
double? LatencyMs,
|
||||
DateTimeOffset? CheckedAt,
|
||||
string? Error);
|
||||
|
||||
internal sealed record ReadyDocument(
|
||||
string Status,
|
||||
DateTimeOffset StartedAt,
|
||||
double UptimeSeconds,
|
||||
MongoReadyHealth Mongo);
|
||||
@@ -0,0 +1,22 @@
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Diagnostics;
|
||||
|
||||
internal static class IngestionMetrics
|
||||
{
|
||||
internal const string MeterName = "StellaOps.Concelier.WebService.Ingestion";
|
||||
|
||||
private static readonly Meter Meter = new(MeterName);
|
||||
|
||||
internal static readonly Counter<long> WriteCounter = Meter.CreateCounter<long>(
|
||||
"ingestion_write_total",
|
||||
description: "Counts raw advisory ingestion attempts, segmented by tenant, source, and result.");
|
||||
|
||||
internal static readonly Counter<long> ViolationCounter = Meter.CreateCounter<long>(
|
||||
"aoc_violation_total",
|
||||
description: "Counts Aggregation-Only Contract violations detected during ingestion.");
|
||||
|
||||
internal static readonly Counter<long> VerificationCounter = Meter.CreateCounter<long>(
|
||||
"verify_runs_total",
|
||||
description: "Counts AOC verification runs initiated via the API.");
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Diagnostics;
|
||||
|
||||
internal static class JobMetrics
|
||||
{
|
||||
internal const string MeterName = "StellaOps.Concelier.WebService.Jobs";
|
||||
|
||||
private static readonly Meter Meter = new(MeterName);
|
||||
|
||||
internal static readonly Counter<long> TriggerCounter = Meter.CreateCounter<long>(
|
||||
"web.jobs.triggered",
|
||||
unit: "count",
|
||||
description: "Number of job trigger requests accepted by the web service.");
|
||||
|
||||
internal static readonly Counter<long> TriggerConflictCounter = Meter.CreateCounter<long>(
|
||||
"web.jobs.trigger.conflict",
|
||||
unit: "count",
|
||||
description: "Number of job trigger requests that resulted in conflicts or rejections.");
|
||||
|
||||
internal static readonly Counter<long> TriggerFailureCounter = Meter.CreateCounter<long>(
|
||||
"web.jobs.trigger.failed",
|
||||
unit: "count",
|
||||
description: "Number of job trigger requests that failed at runtime.");
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
namespace StellaOps.Concelier.WebService.Diagnostics;
|
||||
|
||||
internal static class ProblemTypes
|
||||
{
|
||||
public const string NotFound = "https://stellaops.org/problems/not-found";
|
||||
public const string Validation = "https://stellaops.org/problems/validation";
|
||||
public const string Conflict = "https://stellaops.org/problems/conflict";
|
||||
public const string Locked = "https://stellaops.org/problems/locked";
|
||||
public const string LeaseRejected = "https://stellaops.org/problems/lease-rejected";
|
||||
public const string JobFailure = "https://stellaops.org/problems/job-failure";
|
||||
public const string ServiceUnavailable = "https://stellaops.org/problems/service-unavailable";
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Diagnostics;
|
||||
|
||||
internal sealed class ServiceStatus
|
||||
{
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly DateTimeOffset _startedAt;
|
||||
private readonly object _sync = new();
|
||||
|
||||
private DateTimeOffset? _bootstrapCompletedAt;
|
||||
private TimeSpan? _bootstrapDuration;
|
||||
private DateTimeOffset? _lastReadyCheckAt;
|
||||
private TimeSpan? _lastMongoLatency;
|
||||
private string? _lastMongoError;
|
||||
private bool _lastReadySucceeded;
|
||||
|
||||
public ServiceStatus(TimeProvider timeProvider)
|
||||
{
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_startedAt = _timeProvider.GetUtcNow();
|
||||
}
|
||||
|
||||
public ServiceHealthSnapshot CreateSnapshot()
|
||||
{
|
||||
lock (_sync)
|
||||
{
|
||||
return new ServiceHealthSnapshot(
|
||||
CapturedAt: _timeProvider.GetUtcNow(),
|
||||
StartedAt: _startedAt,
|
||||
BootstrapCompletedAt: _bootstrapCompletedAt,
|
||||
BootstrapDuration: _bootstrapDuration,
|
||||
LastReadyCheckAt: _lastReadyCheckAt,
|
||||
LastMongoLatency: _lastMongoLatency,
|
||||
LastMongoError: _lastMongoError,
|
||||
LastReadySucceeded: _lastReadySucceeded);
|
||||
}
|
||||
}
|
||||
|
||||
public void MarkBootstrapCompleted(TimeSpan duration)
|
||||
{
|
||||
lock (_sync)
|
||||
{
|
||||
var completedAt = _timeProvider.GetUtcNow();
|
||||
_bootstrapCompletedAt = completedAt;
|
||||
_bootstrapDuration = duration;
|
||||
_lastReadySucceeded = true;
|
||||
_lastMongoLatency = duration;
|
||||
_lastMongoError = null;
|
||||
_lastReadyCheckAt = completedAt;
|
||||
}
|
||||
}
|
||||
|
||||
public void RecordMongoCheck(bool success, TimeSpan latency, string? error)
|
||||
{
|
||||
lock (_sync)
|
||||
{
|
||||
_lastReadySucceeded = success;
|
||||
_lastMongoLatency = latency;
|
||||
_lastMongoError = success ? null : error;
|
||||
_lastReadyCheckAt = _timeProvider.GetUtcNow();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record ServiceHealthSnapshot(
|
||||
DateTimeOffset CapturedAt,
|
||||
DateTimeOffset StartedAt,
|
||||
DateTimeOffset? BootstrapCompletedAt,
|
||||
TimeSpan? BootstrapDuration,
|
||||
DateTimeOffset? LastReadyCheckAt,
|
||||
TimeSpan? LastMongoLatency,
|
||||
string? LastMongoError,
|
||||
bool LastReadySucceeded);
|
||||
@@ -0,0 +1,157 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json;
|
||||
using StellaOps.Concelier.RawModels;
|
||||
using StellaOps.Concelier.WebService.Contracts;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Extensions;
|
||||
|
||||
internal static class AdvisoryRawRequestMapper
|
||||
{
|
||||
internal static AdvisoryRawDocument Map(AdvisoryIngestRequest request, string tenant, TimeProvider timeProvider)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
|
||||
ArgumentNullException.ThrowIfNull(timeProvider);
|
||||
|
||||
var sourceRequest = request.Source ?? throw new ArgumentException("source section is required.", nameof(request));
|
||||
var upstreamRequest = request.Upstream ?? throw new ArgumentException("upstream section is required.", nameof(request));
|
||||
var contentRequest = request.Content ?? throw new ArgumentException("content section is required.", nameof(request));
|
||||
var identifiersRequest = request.Identifiers ?? throw new ArgumentException("identifiers section is required.", nameof(request));
|
||||
|
||||
var source = new RawSourceMetadata(
|
||||
sourceRequest.Vendor,
|
||||
sourceRequest.Connector,
|
||||
sourceRequest.Version,
|
||||
string.IsNullOrWhiteSpace(sourceRequest.Stream) ? null : sourceRequest.Stream);
|
||||
|
||||
var signatureRequest = upstreamRequest.Signature ?? new AdvisorySignatureRequest(false, null, null, null, null, null);
|
||||
var signature = new RawSignatureMetadata(
|
||||
signatureRequest.Present,
|
||||
string.IsNullOrWhiteSpace(signatureRequest.Format) ? null : signatureRequest.Format,
|
||||
string.IsNullOrWhiteSpace(signatureRequest.KeyId) ? null : signatureRequest.KeyId,
|
||||
string.IsNullOrWhiteSpace(signatureRequest.Signature) ? null : signatureRequest.Signature,
|
||||
string.IsNullOrWhiteSpace(signatureRequest.Certificate) ? null : signatureRequest.Certificate,
|
||||
string.IsNullOrWhiteSpace(signatureRequest.Digest) ? null : signatureRequest.Digest);
|
||||
|
||||
var retrievedAt = upstreamRequest.RetrievedAt ?? timeProvider.GetUtcNow();
|
||||
|
||||
var upstream = new RawUpstreamMetadata(
|
||||
upstreamRequest.UpstreamId,
|
||||
string.IsNullOrWhiteSpace(upstreamRequest.DocumentVersion) ? null : upstreamRequest.DocumentVersion,
|
||||
retrievedAt,
|
||||
upstreamRequest.ContentHash,
|
||||
signature,
|
||||
NormalizeDictionary(upstreamRequest.Provenance));
|
||||
|
||||
var rawContent = NormalizeRawContent(contentRequest.Raw);
|
||||
var content = new RawContent(
|
||||
contentRequest.Format,
|
||||
string.IsNullOrWhiteSpace(contentRequest.SpecVersion) ? null : contentRequest.SpecVersion,
|
||||
rawContent,
|
||||
string.IsNullOrWhiteSpace(contentRequest.Encoding) ? null : contentRequest.Encoding);
|
||||
|
||||
var aliases = NormalizeStrings(identifiersRequest.Aliases);
|
||||
if (aliases.IsDefault)
|
||||
{
|
||||
aliases = ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
var identifiers = new RawIdentifiers(
|
||||
aliases,
|
||||
identifiersRequest.Primary);
|
||||
|
||||
var linksetRequest = request.Linkset;
|
||||
var linkset = new RawLinkset
|
||||
{
|
||||
Aliases = NormalizeStrings(linksetRequest?.Aliases),
|
||||
PackageUrls = NormalizeStrings(linksetRequest?.PackageUrls),
|
||||
Cpes = NormalizeStrings(linksetRequest?.Cpes),
|
||||
References = NormalizeReferences(linksetRequest?.References),
|
||||
ReconciledFrom = NormalizeStrings(linksetRequest?.ReconciledFrom),
|
||||
Notes = NormalizeDictionary(linksetRequest?.Notes)
|
||||
};
|
||||
|
||||
return new AdvisoryRawDocument(
|
||||
tenant.Trim().ToLowerInvariant(),
|
||||
source,
|
||||
upstream,
|
||||
content,
|
||||
identifiers,
|
||||
linkset);
|
||||
}
|
||||
|
||||
internal static ImmutableArray<string> NormalizeStrings(IEnumerable<string>? values)
|
||||
{
|
||||
if (values is null)
|
||||
{
|
||||
return ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
var builder = ImmutableArray.CreateBuilder<string>();
|
||||
foreach (var value in values)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
builder.Add(value.Trim());
|
||||
}
|
||||
|
||||
return builder.Count == 0 ? ImmutableArray<string>.Empty : builder.ToImmutable();
|
||||
}
|
||||
|
||||
internal static ImmutableDictionary<string, string> NormalizeDictionary(IDictionary<string, string>? values)
|
||||
{
|
||||
if (values is null || values.Count == 0)
|
||||
{
|
||||
return ImmutableDictionary<string, string>.Empty;
|
||||
}
|
||||
|
||||
var builder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
|
||||
foreach (var kv in values)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(kv.Key))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
builder[kv.Key.Trim()] = kv.Value?.Trim() ?? string.Empty;
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
private static ImmutableArray<RawReference> NormalizeReferences(IEnumerable<AdvisoryLinksetReferenceRequest>? references)
|
||||
{
|
||||
if (references is null)
|
||||
{
|
||||
return ImmutableArray<RawReference>.Empty;
|
||||
}
|
||||
|
||||
var builder = ImmutableArray.CreateBuilder<RawReference>();
|
||||
foreach (var reference in references)
|
||||
{
|
||||
if (reference is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(reference.Type) || string.IsNullOrWhiteSpace(reference.Url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
builder.Add(new RawReference(reference.Type.Trim(), reference.Url.Trim(), string.IsNullOrWhiteSpace(reference.Source) ? null : reference.Source.Trim()));
|
||||
}
|
||||
|
||||
return builder.Count == 0 ? ImmutableArray<RawReference>.Empty : builder.ToImmutable();
|
||||
}
|
||||
|
||||
private static JsonElement NormalizeRawContent(JsonElement element)
|
||||
{
|
||||
var json = element.ValueKind == JsonValueKind.Undefined ? "{}" : element.GetRawText();
|
||||
using var document = JsonDocument.Parse(string.IsNullOrWhiteSpace(json) ? "{}" : json);
|
||||
return document.RootElement.Clone();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using YamlDotNet.Serialization;
|
||||
using YamlDotNet.Serialization.NamingConventions;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Extensions;
|
||||
|
||||
public static class ConfigurationExtensions
|
||||
{
|
||||
public static IConfigurationBuilder AddConcelierYaml(this IConfigurationBuilder builder, string path)
|
||||
{
|
||||
if (builder is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(builder));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(path) || !File.Exists(path))
|
||||
{
|
||||
return builder;
|
||||
}
|
||||
|
||||
var deserializer = new DeserializerBuilder()
|
||||
.WithNamingConvention(CamelCaseNamingConvention.Instance)
|
||||
.Build();
|
||||
|
||||
using var reader = File.OpenText(path);
|
||||
var yamlObject = deserializer.Deserialize(reader);
|
||||
if (yamlObject is null)
|
||||
{
|
||||
return builder;
|
||||
}
|
||||
|
||||
var json = JsonSerializer.Serialize(yamlObject);
|
||||
var stream = new MemoryStream(Encoding.UTF8.GetBytes(json));
|
||||
return builder.AddJsonStream(stream);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Merge.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Extensions;
|
||||
|
||||
internal static class JobRegistrationExtensions
|
||||
{
|
||||
private sealed record BuiltInJob(
|
||||
string Kind,
|
||||
string JobType,
|
||||
string AssemblyName,
|
||||
TimeSpan Timeout,
|
||||
TimeSpan LeaseDuration,
|
||||
string? CronExpression = null);
|
||||
|
||||
private static readonly IReadOnlyList<BuiltInJob> BuiltInJobs = new List<BuiltInJob>
|
||||
{
|
||||
new("source:redhat:fetch", "StellaOps.Concelier.Connector.Distro.RedHat.RedHatFetchJob", "StellaOps.Concelier.Connector.Distro.RedHat", TimeSpan.FromMinutes(12), TimeSpan.FromMinutes(6), "0,15,30,45 * * * *"),
|
||||
new("source:redhat:parse", "StellaOps.Concelier.Connector.Distro.RedHat.RedHatParseJob", "StellaOps.Concelier.Connector.Distro.RedHat", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(6), "5,20,35,50 * * * *"),
|
||||
new("source:redhat:map", "StellaOps.Concelier.Connector.Distro.RedHat.RedHatMapJob", "StellaOps.Concelier.Connector.Distro.RedHat", TimeSpan.FromMinutes(20), TimeSpan.FromMinutes(6), "10,25,40,55 * * * *"),
|
||||
|
||||
new("source:cert-in:fetch", "StellaOps.Concelier.Connector.CertIn.CertInFetchJob", "StellaOps.Concelier.Connector.CertIn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)),
|
||||
new("source:cert-in:parse", "StellaOps.Concelier.Connector.CertIn.CertInParseJob", "StellaOps.Concelier.Connector.CertIn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)),
|
||||
new("source:cert-in:map", "StellaOps.Concelier.Connector.CertIn.CertInMapJob", "StellaOps.Concelier.Connector.CertIn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)),
|
||||
|
||||
new("source:cert-fr:fetch", "StellaOps.Concelier.Connector.CertFr.CertFrFetchJob", "StellaOps.Concelier.Connector.CertFr", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)),
|
||||
new("source:cert-fr:parse", "StellaOps.Concelier.Connector.CertFr.CertFrParseJob", "StellaOps.Concelier.Connector.CertFr", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)),
|
||||
new("source:cert-fr:map", "StellaOps.Concelier.Connector.CertFr.CertFrMapJob", "StellaOps.Concelier.Connector.CertFr", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)),
|
||||
|
||||
new("source:jvn:fetch", "StellaOps.Concelier.Connector.Jvn.JvnFetchJob", "StellaOps.Concelier.Connector.Jvn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)),
|
||||
new("source:jvn:parse", "StellaOps.Concelier.Connector.Jvn.JvnParseJob", "StellaOps.Concelier.Connector.Jvn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)),
|
||||
new("source:jvn:map", "StellaOps.Concelier.Connector.Jvn.JvnMapJob", "StellaOps.Concelier.Connector.Jvn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)),
|
||||
|
||||
new("source:ics-kaspersky:fetch", "StellaOps.Concelier.Connector.Ics.Kaspersky.KasperskyFetchJob", "StellaOps.Concelier.Connector.Ics.Kaspersky", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)),
|
||||
new("source:ics-kaspersky:parse", "StellaOps.Concelier.Connector.Ics.Kaspersky.KasperskyParseJob", "StellaOps.Concelier.Connector.Ics.Kaspersky", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)),
|
||||
new("source:ics-kaspersky:map", "StellaOps.Concelier.Connector.Ics.Kaspersky.KasperskyMapJob", "StellaOps.Concelier.Connector.Ics.Kaspersky", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)),
|
||||
|
||||
new("source:osv:fetch", "StellaOps.Concelier.Connector.Osv.OsvFetchJob", "StellaOps.Concelier.Connector.Osv", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)),
|
||||
new("source:osv:parse", "StellaOps.Concelier.Connector.Osv.OsvParseJob", "StellaOps.Concelier.Connector.Osv", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)),
|
||||
new("source:osv:map", "StellaOps.Concelier.Connector.Osv.OsvMapJob", "StellaOps.Concelier.Connector.Osv", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)),
|
||||
|
||||
new("source:vmware:fetch", "StellaOps.Concelier.Connector.Vndr.Vmware.VmwareFetchJob", "StellaOps.Concelier.Connector.Vndr.Vmware", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)),
|
||||
new("source:vmware:parse", "StellaOps.Concelier.Connector.Vndr.Vmware.VmwareParseJob", "StellaOps.Concelier.Connector.Vndr.Vmware", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)),
|
||||
new("source:vmware:map", "StellaOps.Concelier.Connector.Vndr.Vmware.VmwareMapJob", "StellaOps.Concelier.Connector.Vndr.Vmware", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)),
|
||||
|
||||
new("source:vndr-oracle:fetch", "StellaOps.Concelier.Connector.Vndr.Oracle.OracleFetchJob", "StellaOps.Concelier.Connector.Vndr.Oracle", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)),
|
||||
new("source:vndr-oracle:parse", "StellaOps.Concelier.Connector.Vndr.Oracle.OracleParseJob", "StellaOps.Concelier.Connector.Vndr.Oracle", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)),
|
||||
new("source:vndr-oracle:map", "StellaOps.Concelier.Connector.Vndr.Oracle.OracleMapJob", "StellaOps.Concelier.Connector.Vndr.Oracle", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)),
|
||||
|
||||
new("export:json", "StellaOps.Concelier.Exporter.Json.JsonExportJob", "StellaOps.Concelier.Exporter.Json", TimeSpan.FromMinutes(10), TimeSpan.FromMinutes(5)),
|
||||
new("export:trivy-db", "StellaOps.Concelier.Exporter.TrivyDb.TrivyDbExportJob", "StellaOps.Concelier.Exporter.TrivyDb", TimeSpan.FromMinutes(20), TimeSpan.FromMinutes(10)),
|
||||
new("merge:reconcile", "StellaOps.Concelier.Merge.Jobs.MergeReconcileJob", "StellaOps.Concelier.Merge", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5))
|
||||
};
|
||||
|
||||
public static IServiceCollection AddBuiltInConcelierJobs(this IServiceCollection services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
foreach (var registration in BuiltInJobs)
|
||||
{
|
||||
if (options.Definitions.ContainsKey(registration.Kind))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var jobType = Type.GetType(
|
||||
$"{registration.JobType}, {registration.AssemblyName}",
|
||||
throwOnError: false,
|
||||
ignoreCase: false);
|
||||
|
||||
if (jobType is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var timeout = registration.Timeout > TimeSpan.Zero ? registration.Timeout : options.DefaultTimeout;
|
||||
var lease = registration.LeaseDuration > TimeSpan.Zero ? registration.LeaseDuration : options.DefaultLeaseDuration;
|
||||
|
||||
options.Definitions[registration.Kind] = new JobDefinition(
|
||||
registration.Kind,
|
||||
jobType,
|
||||
timeout,
|
||||
lease,
|
||||
registration.CronExpression,
|
||||
Enabled: true);
|
||||
}
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.WebService.Options;
|
||||
using StellaOps.Concelier.WebService.Services;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Extensions;
|
||||
|
||||
internal static class MirrorEndpointExtensions
|
||||
{
|
||||
private const string IndexScope = "index";
|
||||
private const string DownloadScope = "download";
|
||||
|
||||
public static void MapConcelierMirrorEndpoints(this WebApplication app, bool authorityConfigured, bool enforceAuthority)
|
||||
{
|
||||
app.MapGet("/concelier/exports/index.json", async (
|
||||
MirrorFileLocator locator,
|
||||
MirrorRateLimiter limiter,
|
||||
IOptionsMonitor<ConcelierOptions> optionsMonitor,
|
||||
HttpContext context,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var mirrorOptions = optionsMonitor.CurrentValue.Mirror ?? new ConcelierOptions.MirrorOptions();
|
||||
if (!mirrorOptions.Enabled)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
if (!TryAuthorize(mirrorOptions.RequireAuthentication, enforceAuthority, context, authorityConfigured, out var unauthorizedResult))
|
||||
{
|
||||
return unauthorizedResult;
|
||||
}
|
||||
|
||||
if (!limiter.TryAcquire("__index__", IndexScope, mirrorOptions.MaxIndexRequestsPerHour, out var retryAfter))
|
||||
{
|
||||
ApplyRetryAfter(context.Response, retryAfter);
|
||||
return Results.StatusCode(StatusCodes.Status429TooManyRequests);
|
||||
}
|
||||
|
||||
if (!locator.TryResolveIndex(out var path, out _))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
return await WriteFileAsync(path, context.Response, "application/json").ConfigureAwait(false);
|
||||
});
|
||||
|
||||
app.MapGet("/concelier/exports/{**relativePath}", async (
|
||||
string? relativePath,
|
||||
MirrorFileLocator locator,
|
||||
MirrorRateLimiter limiter,
|
||||
IOptionsMonitor<ConcelierOptions> optionsMonitor,
|
||||
HttpContext context,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var mirrorOptions = optionsMonitor.CurrentValue.Mirror ?? new ConcelierOptions.MirrorOptions();
|
||||
if (!mirrorOptions.Enabled)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(relativePath))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
if (!locator.TryResolveRelativePath(relativePath, out var path, out _, out var domainId))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
var domain = FindDomain(mirrorOptions, domainId);
|
||||
|
||||
if (!TryAuthorize(domain?.RequireAuthentication ?? mirrorOptions.RequireAuthentication, enforceAuthority, context, authorityConfigured, out var unauthorizedResult))
|
||||
{
|
||||
return unauthorizedResult;
|
||||
}
|
||||
|
||||
var limit = domain?.MaxDownloadRequestsPerHour ?? mirrorOptions.MaxIndexRequestsPerHour;
|
||||
if (!limiter.TryAcquire(domain?.Id ?? "__mirror__", DownloadScope, limit, out var retryAfter))
|
||||
{
|
||||
ApplyRetryAfter(context.Response, retryAfter);
|
||||
return Results.StatusCode(StatusCodes.Status429TooManyRequests);
|
||||
}
|
||||
|
||||
var contentType = ResolveContentType(path);
|
||||
return await WriteFileAsync(path, context.Response, contentType).ConfigureAwait(false);
|
||||
});
|
||||
}
|
||||
|
||||
private static ConcelierOptions.MirrorDomainOptions? FindDomain(ConcelierOptions.MirrorOptions mirrorOptions, string? domainId)
|
||||
{
|
||||
if (domainId is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
foreach (var candidate in mirrorOptions.Domains)
|
||||
{
|
||||
if (candidate is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.Equals(candidate.Id, domainId, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool TryAuthorize(bool requireAuthentication, bool enforceAuthority, HttpContext context, bool authorityConfigured, out IResult result)
|
||||
{
|
||||
result = Results.Empty;
|
||||
if (!requireAuthentication)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!enforceAuthority || !authorityConfigured)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (context.User?.Identity?.IsAuthenticated == true)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
context.Response.Headers.WWWAuthenticate = "Bearer realm=\"StellaOps Concelier Mirror\"";
|
||||
result = Results.StatusCode(StatusCodes.Status401Unauthorized);
|
||||
return false;
|
||||
}
|
||||
|
||||
private static Task<IResult> WriteFileAsync(string path, HttpResponse response, string contentType)
|
||||
{
|
||||
var fileInfo = new FileInfo(path);
|
||||
if (!fileInfo.Exists)
|
||||
{
|
||||
return Task.FromResult(Results.NotFound());
|
||||
}
|
||||
|
||||
var stream = new FileStream(
|
||||
path,
|
||||
FileMode.Open,
|
||||
FileAccess.Read,
|
||||
FileShare.Read | FileShare.Delete);
|
||||
|
||||
response.Headers.CacheControl = BuildCacheControlHeader(path);
|
||||
response.Headers.LastModified = fileInfo.LastWriteTimeUtc.ToString("R", CultureInfo.InvariantCulture);
|
||||
response.ContentLength = fileInfo.Length;
|
||||
return Task.FromResult(Results.Stream(stream, contentType));
|
||||
}
|
||||
|
||||
private static string ResolveContentType(string path)
|
||||
{
|
||||
if (path.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "application/json";
|
||||
}
|
||||
|
||||
if (path.EndsWith(".jws", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "application/jose+json";
|
||||
}
|
||||
|
||||
return "application/octet-stream";
|
||||
}
|
||||
|
||||
private static void ApplyRetryAfter(HttpResponse response, TimeSpan? retryAfter)
|
||||
{
|
||||
if (retryAfter is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var seconds = Math.Max((int)Math.Ceiling(retryAfter.Value.TotalSeconds), 1);
|
||||
response.Headers.RetryAfter = seconds.ToString(CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
private static string BuildCacheControlHeader(string path)
|
||||
{
|
||||
var fileName = Path.GetFileName(path);
|
||||
if (fileName is null)
|
||||
{
|
||||
return "public, max-age=60";
|
||||
}
|
||||
|
||||
if (string.Equals(fileName, "index.json", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "public, max-age=60";
|
||||
}
|
||||
|
||||
if (fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase) ||
|
||||
fileName.EndsWith(".jws", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "public, max-age=300, immutable";
|
||||
}
|
||||
|
||||
return "public, max-age=300";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using OpenTelemetry.Metrics;
|
||||
using OpenTelemetry.Resources;
|
||||
using OpenTelemetry.Trace;
|
||||
using Serilog;
|
||||
using Serilog.Core;
|
||||
using Serilog.Events;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.Common.Telemetry;
|
||||
using StellaOps.Concelier.WebService.Diagnostics;
|
||||
using StellaOps.Concelier.WebService.Options;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Extensions;
|
||||
|
||||
public static class TelemetryExtensions
|
||||
{
|
||||
public static void ConfigureConcelierTelemetry(this WebApplicationBuilder builder, ConcelierOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(builder);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
var telemetry = options.Telemetry ?? new ConcelierOptions.TelemetryOptions();
|
||||
|
||||
if (telemetry.EnableLogging)
|
||||
{
|
||||
builder.Host.UseSerilog((context, services, configuration) =>
|
||||
{
|
||||
ConfigureSerilog(configuration, telemetry, builder.Environment.EnvironmentName, builder.Environment.ApplicationName);
|
||||
});
|
||||
}
|
||||
|
||||
if (!telemetry.Enabled || (!telemetry.EnableTracing && !telemetry.EnableMetrics))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var openTelemetry = builder.Services.AddOpenTelemetry();
|
||||
|
||||
openTelemetry.ConfigureResource(resource =>
|
||||
{
|
||||
var serviceName = telemetry.ServiceName ?? builder.Environment.ApplicationName;
|
||||
var version = Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown";
|
||||
|
||||
resource.AddService(serviceName, serviceVersion: version, serviceInstanceId: Environment.MachineName);
|
||||
resource.AddAttributes(new[]
|
||||
{
|
||||
new KeyValuePair<string, object>("deployment.environment", builder.Environment.EnvironmentName),
|
||||
});
|
||||
|
||||
foreach (var attribute in telemetry.ResourceAttributes)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(attribute.Key) || attribute.Value is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
resource.AddAttributes(new[] { new KeyValuePair<string, object>(attribute.Key, attribute.Value) });
|
||||
}
|
||||
});
|
||||
|
||||
if (telemetry.EnableTracing)
|
||||
{
|
||||
openTelemetry.WithTracing(tracing =>
|
||||
{
|
||||
tracing
|
||||
.AddSource(JobDiagnostics.ActivitySourceName)
|
||||
.AddSource(SourceDiagnostics.ActivitySourceName)
|
||||
.AddAspNetCoreInstrumentation()
|
||||
.AddHttpClientInstrumentation();
|
||||
|
||||
ConfigureExporters(telemetry, tracing);
|
||||
});
|
||||
}
|
||||
|
||||
if (telemetry.EnableMetrics)
|
||||
{
|
||||
openTelemetry.WithMetrics(metrics =>
|
||||
{
|
||||
metrics
|
||||
.AddMeter(JobDiagnostics.MeterName)
|
||||
.AddMeter(SourceDiagnostics.MeterName)
|
||||
.AddMeter(IngestionMetrics.MeterName)
|
||||
.AddMeter("StellaOps.Concelier.Connector.CertBund")
|
||||
.AddMeter("StellaOps.Concelier.Connector.Nvd")
|
||||
.AddMeter("StellaOps.Concelier.Connector.Vndr.Chromium")
|
||||
.AddMeter("StellaOps.Concelier.Connector.Vndr.Apple")
|
||||
.AddMeter("StellaOps.Concelier.Connector.Vndr.Adobe")
|
||||
.AddMeter(JobMetrics.MeterName)
|
||||
.AddAspNetCoreInstrumentation()
|
||||
.AddHttpClientInstrumentation()
|
||||
.AddRuntimeInstrumentation();
|
||||
|
||||
ConfigureExporters(telemetry, metrics);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private static void ConfigureSerilog(LoggerConfiguration configuration, ConcelierOptions.TelemetryOptions telemetry, string environmentName, string applicationName)
|
||||
{
|
||||
if (!Enum.TryParse(telemetry.MinimumLogLevel, ignoreCase: true, out LogEventLevel level))
|
||||
{
|
||||
level = LogEventLevel.Information;
|
||||
}
|
||||
|
||||
configuration
|
||||
.MinimumLevel.Is(level)
|
||||
.MinimumLevel.Override("Microsoft", LogEventLevel.Warning)
|
||||
.MinimumLevel.Override("Microsoft.Hosting.Lifetime", LogEventLevel.Information)
|
||||
.Enrich.FromLogContext()
|
||||
.Enrich.With<ActivityEnricher>()
|
||||
.Enrich.WithProperty("service.name", telemetry.ServiceName ?? applicationName)
|
||||
.Enrich.WithProperty("deployment.environment", environmentName)
|
||||
.WriteTo.Console(outputTemplate: "[{Timestamp:O}] [{Level:u3}] {Message:lj} {Properties}{NewLine}{Exception}");
|
||||
}
|
||||
|
||||
private static void ConfigureExporters(ConcelierOptions.TelemetryOptions telemetry, TracerProviderBuilder tracing)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(telemetry.OtlpEndpoint))
|
||||
{
|
||||
if (telemetry.ExportConsole)
|
||||
{
|
||||
tracing.AddConsoleExporter();
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
tracing.AddOtlpExporter(options =>
|
||||
{
|
||||
options.Endpoint = new Uri(telemetry.OtlpEndpoint);
|
||||
var headers = BuildHeaders(telemetry);
|
||||
if (!string.IsNullOrEmpty(headers))
|
||||
{
|
||||
options.Headers = headers;
|
||||
}
|
||||
});
|
||||
|
||||
if (telemetry.ExportConsole)
|
||||
{
|
||||
tracing.AddConsoleExporter();
|
||||
}
|
||||
}
|
||||
|
||||
private static void ConfigureExporters(ConcelierOptions.TelemetryOptions telemetry, MeterProviderBuilder metrics)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(telemetry.OtlpEndpoint))
|
||||
{
|
||||
if (telemetry.ExportConsole)
|
||||
{
|
||||
metrics.AddConsoleExporter();
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
metrics.AddOtlpExporter(options =>
|
||||
{
|
||||
options.Endpoint = new Uri(telemetry.OtlpEndpoint);
|
||||
var headers = BuildHeaders(telemetry);
|
||||
if (!string.IsNullOrEmpty(headers))
|
||||
{
|
||||
options.Headers = headers;
|
||||
}
|
||||
});
|
||||
|
||||
if (telemetry.ExportConsole)
|
||||
{
|
||||
metrics.AddConsoleExporter();
|
||||
}
|
||||
}
|
||||
|
||||
private static string? BuildHeaders(ConcelierOptions.TelemetryOptions telemetry)
|
||||
{
|
||||
if (telemetry.OtlpHeaders.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return string.Join(",", telemetry.OtlpHeaders
|
||||
.Where(static kvp => !string.IsNullOrWhiteSpace(kvp.Key) && !string.IsNullOrWhiteSpace(kvp.Value))
|
||||
.Select(static kvp => $"{kvp.Key}={kvp.Value}"));
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class ActivityEnricher : ILogEventEnricher
|
||||
{
|
||||
public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory)
|
||||
{
|
||||
var activity = Activity.Current;
|
||||
if (activity is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (activity.TraceId != default)
|
||||
{
|
||||
logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty("trace_id", activity.TraceId.ToString()));
|
||||
}
|
||||
|
||||
if (activity.SpanId != default)
|
||||
{
|
||||
logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty("span_id", activity.SpanId.ToString()));
|
||||
}
|
||||
|
||||
if (activity.ParentSpanId != default)
|
||||
{
|
||||
logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty("parent_span_id", activity.ParentSpanId.ToString()));
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(activity.TraceStateString))
|
||||
{
|
||||
logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty("trace_state", activity.TraceStateString));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Security.Claims;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Auth.Abstractions;
|
||||
using StellaOps.Concelier.WebService.Options;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Filters;
|
||||
|
||||
/// <summary>
|
||||
/// Emits structured audit logs for job endpoint authorization decisions, including bypass usage.
|
||||
/// </summary>
|
||||
public sealed class JobAuthorizationAuditFilter : IEndpointFilter
|
||||
{
|
||||
internal const string LoggerName = "Concelier.Authorization.Audit";
|
||||
|
||||
public async ValueTask<object?> InvokeAsync(EndpointFilterInvocationContext context, EndpointFilterDelegate next)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
ArgumentNullException.ThrowIfNull(next);
|
||||
|
||||
var httpContext = context.HttpContext;
|
||||
var options = httpContext.RequestServices.GetRequiredService<IOptions<ConcelierOptions>>().Value;
|
||||
var authority = options.Authority;
|
||||
|
||||
if (authority is null || !authority.Enabled)
|
||||
{
|
||||
return await next(context).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var logger = httpContext.RequestServices
|
||||
.GetRequiredService<ILoggerFactory>()
|
||||
.CreateLogger(LoggerName);
|
||||
|
||||
var remoteAddress = httpContext.Connection.RemoteIpAddress;
|
||||
var matcher = new NetworkMaskMatcher(authority.BypassNetworks);
|
||||
var user = httpContext.User;
|
||||
var isAuthenticated = user?.Identity?.IsAuthenticated ?? false;
|
||||
var bypassUsed = !isAuthenticated && matcher.IsAllowed(remoteAddress);
|
||||
|
||||
var result = await next(context).ConfigureAwait(false);
|
||||
|
||||
var scopes = ExtractScopes(user);
|
||||
var subject = user?.FindFirst(StellaOpsClaimTypes.Subject)?.Value;
|
||||
var clientId = user?.FindFirst(StellaOpsClaimTypes.ClientId)?.Value;
|
||||
|
||||
logger.LogInformation(
|
||||
"Concelier authorization audit route={Route} status={StatusCode} subject={Subject} clientId={ClientId} scopes={Scopes} bypass={Bypass} remote={RemoteAddress}",
|
||||
httpContext.Request.Path.Value ?? string.Empty,
|
||||
httpContext.Response.StatusCode,
|
||||
string.IsNullOrWhiteSpace(subject) ? "(anonymous)" : subject,
|
||||
string.IsNullOrWhiteSpace(clientId) ? "(none)" : clientId,
|
||||
scopes.Length == 0 ? "(none)" : string.Join(',', scopes),
|
||||
bypassUsed,
|
||||
remoteAddress?.ToString() ?? IPAddress.None.ToString());
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static string[] ExtractScopes(ClaimsPrincipal? principal)
|
||||
{
|
||||
if (principal is null)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var values = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var claim in principal.FindAll(StellaOpsClaimTypes.ScopeItem))
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(claim.Value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
values.Add(claim.Value);
|
||||
}
|
||||
|
||||
foreach (var claim in principal.FindAll(StellaOpsClaimTypes.Scope))
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(claim.Value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var parts = claim.Value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
foreach (var part in parts)
|
||||
{
|
||||
var normalized = StellaOpsScopes.Normalize(part);
|
||||
if (!string.IsNullOrEmpty(normalized))
|
||||
{
|
||||
values.Add(normalized);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return values.Count == 0 ? Array.Empty<string>() : values.ToArray();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Jobs;
|
||||
|
||||
public sealed record JobDefinitionResponse(
|
||||
string Kind,
|
||||
bool Enabled,
|
||||
string? CronExpression,
|
||||
TimeSpan Timeout,
|
||||
TimeSpan LeaseDuration,
|
||||
JobRunResponse? LastRun)
|
||||
{
|
||||
public static JobDefinitionResponse FromDefinition(JobDefinition definition, JobRunSnapshot? lastRun)
|
||||
{
|
||||
return new JobDefinitionResponse(
|
||||
definition.Kind,
|
||||
definition.Enabled,
|
||||
definition.CronExpression,
|
||||
definition.Timeout,
|
||||
definition.LeaseDuration,
|
||||
lastRun is null ? null : JobRunResponse.FromSnapshot(lastRun));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Jobs;
|
||||
|
||||
public sealed record JobRunResponse(
|
||||
Guid RunId,
|
||||
string Kind,
|
||||
JobRunStatus Status,
|
||||
string Trigger,
|
||||
DateTimeOffset CreatedAt,
|
||||
DateTimeOffset? StartedAt,
|
||||
DateTimeOffset? CompletedAt,
|
||||
string? Error,
|
||||
TimeSpan? Duration,
|
||||
IReadOnlyDictionary<string, object?> Parameters)
|
||||
{
|
||||
public static JobRunResponse FromSnapshot(JobRunSnapshot snapshot)
|
||||
=> new(
|
||||
snapshot.RunId,
|
||||
snapshot.Kind,
|
||||
snapshot.Status,
|
||||
snapshot.Trigger,
|
||||
snapshot.CreatedAt,
|
||||
snapshot.StartedAt,
|
||||
snapshot.CompletedAt,
|
||||
snapshot.Error,
|
||||
snapshot.Duration,
|
||||
snapshot.Parameters);
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace StellaOps.Concelier.WebService.Jobs;
|
||||
|
||||
public sealed class JobTriggerRequest
|
||||
{
|
||||
public string Trigger { get; set; } = "api";
|
||||
|
||||
public Dictionary<string, object?> Parameters { get; set; } = new(StringComparer.Ordinal);
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Options;
|
||||
|
||||
public sealed class ConcelierOptions
|
||||
{
|
||||
public StorageOptions Storage { get; set; } = new();
|
||||
|
||||
public PluginOptions Plugins { get; set; } = new();
|
||||
|
||||
public TelemetryOptions Telemetry { get; set; } = new();
|
||||
|
||||
public AuthorityOptions Authority { get; set; } = new();
|
||||
|
||||
public MirrorOptions Mirror { get; set; } = new();
|
||||
|
||||
public sealed class StorageOptions
|
||||
{
|
||||
public string Driver { get; set; } = "mongo";
|
||||
|
||||
public string Dsn { get; set; } = string.Empty;
|
||||
|
||||
public string? Database { get; set; }
|
||||
|
||||
public int CommandTimeoutSeconds { get; set; } = 30;
|
||||
}
|
||||
|
||||
public sealed class PluginOptions
|
||||
{
|
||||
public string? BaseDirectory { get; set; }
|
||||
|
||||
public string? Directory { get; set; }
|
||||
|
||||
public IList<string> SearchPatterns { get; set; } = new List<string>();
|
||||
}
|
||||
|
||||
public sealed class TelemetryOptions
|
||||
{
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
public bool EnableTracing { get; set; } = true;
|
||||
|
||||
public bool EnableMetrics { get; set; } = true;
|
||||
|
||||
public bool EnableLogging { get; set; } = true;
|
||||
|
||||
public string MinimumLogLevel { get; set; } = "Information";
|
||||
|
||||
public string? ServiceName { get; set; }
|
||||
|
||||
public string? OtlpEndpoint { get; set; }
|
||||
|
||||
public IDictionary<string, string> OtlpHeaders { get; set; } = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public IDictionary<string, string> ResourceAttributes { get; set; } = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public bool ExportConsole { get; set; }
|
||||
}
|
||||
|
||||
public sealed class AuthorityOptions
|
||||
{
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
public bool AllowAnonymousFallback { get; set; } = true;
|
||||
|
||||
public string Issuer { get; set; } = string.Empty;
|
||||
|
||||
public string? MetadataAddress { get; set; }
|
||||
|
||||
public bool RequireHttpsMetadata { get; set; } = true;
|
||||
|
||||
public int BackchannelTimeoutSeconds { get; set; } = 30;
|
||||
|
||||
public int TokenClockSkewSeconds { get; set; } = 60;
|
||||
|
||||
public IList<string> Audiences { get; set; } = new List<string>();
|
||||
|
||||
public IList<string> RequiredScopes { get; set; } = new List<string>();
|
||||
|
||||
public IList<string> BypassNetworks { get; set; } = new List<string>();
|
||||
|
||||
public string? ClientId { get; set; }
|
||||
|
||||
public string? ClientSecret { get; set; }
|
||||
|
||||
public string? ClientSecretFile { get; set; }
|
||||
|
||||
public IList<string> ClientScopes { get; set; } = new List<string>();
|
||||
|
||||
public ResilienceOptions Resilience { get; set; } = new();
|
||||
|
||||
public sealed class ResilienceOptions
|
||||
{
|
||||
public bool? EnableRetries { get; set; }
|
||||
|
||||
public IList<TimeSpan> RetryDelays { get; set; } = new List<TimeSpan>();
|
||||
|
||||
public bool? AllowOfflineCacheFallback { get; set; }
|
||||
|
||||
public TimeSpan? OfflineCacheTolerance { get; set; }
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class MirrorOptions
|
||||
{
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
public string ExportRoot { get; set; } = System.IO.Path.Combine("exports", "json");
|
||||
|
||||
public string? ActiveExportId { get; set; }
|
||||
|
||||
public string LatestDirectoryName { get; set; } = "latest";
|
||||
|
||||
public string MirrorDirectoryName { get; set; } = "mirror";
|
||||
|
||||
public bool RequireAuthentication { get; set; }
|
||||
|
||||
public int MaxIndexRequestsPerHour { get; set; } = 600;
|
||||
|
||||
public IList<MirrorDomainOptions> Domains { get; } = new List<MirrorDomainOptions>();
|
||||
|
||||
[JsonIgnore]
|
||||
public string ExportRootAbsolute { get; internal set; } = string.Empty;
|
||||
}
|
||||
|
||||
public sealed class MirrorDomainOptions
|
||||
{
|
||||
public string Id { get; set; } = string.Empty;
|
||||
|
||||
public string? DisplayName { get; set; }
|
||||
|
||||
public bool RequireAuthentication { get; set; }
|
||||
|
||||
public int MaxDownloadRequestsPerHour { get; set; } = 1200;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Options;
|
||||
|
||||
/// <summary>
|
||||
/// Post-configuration helpers for <see cref="ConcelierOptions"/>.
|
||||
/// </summary>
|
||||
public static class ConcelierOptionsPostConfigure
|
||||
{
|
||||
/// <summary>
|
||||
/// Applies derived settings that require filesystem access, such as loading client secrets from disk.
|
||||
/// </summary>
|
||||
/// <param name="options">The options to mutate.</param>
|
||||
/// <param name="contentRootPath">Application content root used to resolve relative paths.</param>
|
||||
public static void Apply(ConcelierOptions options, string contentRootPath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
options.Authority ??= new ConcelierOptions.AuthorityOptions();
|
||||
|
||||
var authority = options.Authority;
|
||||
if (string.IsNullOrWhiteSpace(authority.ClientSecret)
|
||||
&& !string.IsNullOrWhiteSpace(authority.ClientSecretFile))
|
||||
{
|
||||
var resolvedPath = authority.ClientSecretFile!;
|
||||
if (!Path.IsPathRooted(resolvedPath))
|
||||
{
|
||||
resolvedPath = Path.Combine(contentRootPath, resolvedPath);
|
||||
}
|
||||
|
||||
if (!File.Exists(resolvedPath))
|
||||
{
|
||||
throw new InvalidOperationException($"Authority client secret file '{resolvedPath}' was not found.");
|
||||
}
|
||||
|
||||
var secret = File.ReadAllText(resolvedPath).Trim();
|
||||
if (string.IsNullOrEmpty(secret))
|
||||
{
|
||||
throw new InvalidOperationException($"Authority client secret file '{resolvedPath}' is empty.");
|
||||
}
|
||||
|
||||
authority.ClientSecret = secret;
|
||||
}
|
||||
|
||||
options.Mirror ??= new ConcelierOptions.MirrorOptions();
|
||||
var mirror = options.Mirror;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(mirror.ExportRoot))
|
||||
{
|
||||
mirror.ExportRoot = Path.Combine("exports", "json");
|
||||
}
|
||||
|
||||
var resolvedRoot = mirror.ExportRoot;
|
||||
if (!Path.IsPathRooted(resolvedRoot))
|
||||
{
|
||||
resolvedRoot = Path.Combine(contentRootPath, resolvedRoot);
|
||||
}
|
||||
|
||||
mirror.ExportRootAbsolute = Path.GetFullPath(resolvedRoot);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(mirror.LatestDirectoryName))
|
||||
{
|
||||
mirror.LatestDirectoryName = "latest";
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(mirror.MirrorDirectoryName))
|
||||
{
|
||||
mirror.MirrorDirectoryName = "mirror";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Auth.Abstractions;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Options;
|
||||
|
||||
public static class ConcelierOptionsValidator
|
||||
{
|
||||
public static void Validate(ConcelierOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
if (!string.Equals(options.Storage.Driver, "mongo", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException("Only Mongo storage driver is supported (storage.driver == 'mongo').");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(options.Storage.Dsn))
|
||||
{
|
||||
throw new InvalidOperationException("Storage DSN must be configured.");
|
||||
}
|
||||
|
||||
if (options.Storage.CommandTimeoutSeconds <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("Command timeout must be greater than zero seconds.");
|
||||
}
|
||||
|
||||
options.Telemetry ??= new ConcelierOptions.TelemetryOptions();
|
||||
|
||||
options.Authority ??= new ConcelierOptions.AuthorityOptions();
|
||||
options.Authority.Resilience ??= new ConcelierOptions.AuthorityOptions.ResilienceOptions();
|
||||
NormalizeList(options.Authority.Audiences, toLower: false);
|
||||
NormalizeList(options.Authority.RequiredScopes, toLower: true);
|
||||
NormalizeList(options.Authority.BypassNetworks, toLower: false);
|
||||
NormalizeList(options.Authority.ClientScopes, toLower: true);
|
||||
ValidateResilience(options.Authority.Resilience);
|
||||
|
||||
if (options.Authority.RequiredScopes.Count == 0)
|
||||
{
|
||||
options.Authority.RequiredScopes.Add(StellaOpsScopes.ConcelierJobsTrigger);
|
||||
}
|
||||
|
||||
if (options.Authority.ClientScopes.Count == 0)
|
||||
{
|
||||
foreach (var scope in options.Authority.RequiredScopes)
|
||||
{
|
||||
options.Authority.ClientScopes.Add(scope);
|
||||
}
|
||||
}
|
||||
|
||||
if (options.Authority.ClientScopes.Count == 0)
|
||||
{
|
||||
options.Authority.ClientScopes.Add(StellaOpsScopes.ConcelierJobsTrigger);
|
||||
}
|
||||
|
||||
if (options.Authority.BackchannelTimeoutSeconds <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("Authority backchannelTimeoutSeconds must be greater than zero.");
|
||||
}
|
||||
|
||||
if (options.Authority.TokenClockSkewSeconds < 0 || options.Authority.TokenClockSkewSeconds > 300)
|
||||
{
|
||||
throw new InvalidOperationException("Authority tokenClockSkewSeconds must be between 0 and 300 seconds.");
|
||||
}
|
||||
|
||||
if (options.Authority.Enabled)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(options.Authority.Issuer))
|
||||
{
|
||||
throw new InvalidOperationException("Authority issuer must be configured when authority is enabled.");
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(options.Authority.Issuer, UriKind.Absolute, out var issuerUri))
|
||||
{
|
||||
throw new InvalidOperationException("Authority issuer must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (options.Authority.RequireHttpsMetadata && !issuerUri.IsLoopback && !string.Equals(issuerUri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException("Authority issuer must use HTTPS when requireHttpsMetadata is enabled.");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.Authority.MetadataAddress) && !Uri.TryCreate(options.Authority.MetadataAddress, UriKind.Absolute, out _))
|
||||
{
|
||||
throw new InvalidOperationException("Authority metadataAddress must be an absolute URI when specified.");
|
||||
}
|
||||
|
||||
if (options.Authority.Audiences.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException("Authority audiences must include at least one entry when authority is enabled.");
|
||||
}
|
||||
|
||||
if (!options.Authority.AllowAnonymousFallback)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(options.Authority.ClientId))
|
||||
{
|
||||
throw new InvalidOperationException("Authority clientId must be configured when anonymous fallback is disabled.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(options.Authority.ClientSecret))
|
||||
{
|
||||
throw new InvalidOperationException("Authority clientSecret must be configured when anonymous fallback is disabled.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!Enum.TryParse(options.Telemetry.MinimumLogLevel, ignoreCase: true, out LogLevel _))
|
||||
{
|
||||
throw new InvalidOperationException($"Telemetry minimum log level '{options.Telemetry.MinimumLogLevel}' is invalid.");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.Telemetry.OtlpEndpoint) && !Uri.TryCreate(options.Telemetry.OtlpEndpoint, UriKind.Absolute, out _))
|
||||
{
|
||||
throw new InvalidOperationException("Telemetry OTLP endpoint must be an absolute URI.");
|
||||
}
|
||||
|
||||
foreach (var attribute in options.Telemetry.ResourceAttributes)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(attribute.Key))
|
||||
{
|
||||
throw new InvalidOperationException("Telemetry resource attribute keys must be non-empty.");
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var header in options.Telemetry.OtlpHeaders)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(header.Key))
|
||||
{
|
||||
throw new InvalidOperationException("Telemetry OTLP header names must be non-empty.");
|
||||
}
|
||||
}
|
||||
|
||||
options.Mirror ??= new ConcelierOptions.MirrorOptions();
|
||||
ValidateMirror(options.Mirror);
|
||||
}
|
||||
|
||||
private static void NormalizeList(IList<string> values, bool toLower)
|
||||
{
|
||||
if (values is null || values.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
for (var index = values.Count - 1; index >= 0; index--)
|
||||
{
|
||||
var entry = values[index];
|
||||
if (string.IsNullOrWhiteSpace(entry))
|
||||
{
|
||||
values.RemoveAt(index);
|
||||
continue;
|
||||
}
|
||||
|
||||
var normalized = entry.Trim();
|
||||
if (toLower)
|
||||
{
|
||||
normalized = normalized.ToLowerInvariant();
|
||||
}
|
||||
|
||||
if (!seen.Add(normalized))
|
||||
{
|
||||
values.RemoveAt(index);
|
||||
continue;
|
||||
}
|
||||
|
||||
values[index] = normalized;
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateResilience(ConcelierOptions.AuthorityOptions.ResilienceOptions resilience)
|
||||
{
|
||||
if (resilience.RetryDelays is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var delay in resilience.RetryDelays)
|
||||
{
|
||||
if (delay <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("Authority resilience retryDelays must be greater than zero.");
|
||||
}
|
||||
}
|
||||
|
||||
if (resilience.OfflineCacheTolerance.HasValue && resilience.OfflineCacheTolerance.Value < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("Authority resilience offlineCacheTolerance must be greater than or equal to zero.");
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateMirror(ConcelierOptions.MirrorOptions mirror)
|
||||
{
|
||||
if (mirror.MaxIndexRequestsPerHour < 0)
|
||||
{
|
||||
throw new InvalidOperationException("Mirror maxIndexRequestsPerHour must be greater than or equal to zero.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(mirror.ExportRoot))
|
||||
{
|
||||
throw new InvalidOperationException("Mirror exportRoot must be configured.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(mirror.ExportRootAbsolute))
|
||||
{
|
||||
throw new InvalidOperationException("Mirror export root could not be resolved.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(mirror.LatestDirectoryName))
|
||||
{
|
||||
throw new InvalidOperationException("Mirror latestDirectoryName must be provided.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(mirror.MirrorDirectoryName))
|
||||
{
|
||||
throw new InvalidOperationException("Mirror mirrorDirectoryName must be provided.");
|
||||
}
|
||||
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var domain in mirror.Domains)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(domain.Id))
|
||||
{
|
||||
throw new InvalidOperationException("Mirror domain id must be provided.");
|
||||
}
|
||||
|
||||
var normalized = domain.Id.Trim();
|
||||
if (!seen.Add(normalized))
|
||||
{
|
||||
throw new InvalidOperationException($"Mirror domain id '{normalized}' is duplicated.");
|
||||
}
|
||||
|
||||
if (domain.MaxDownloadRequestsPerHour < 0)
|
||||
{
|
||||
throw new InvalidOperationException($"Mirror domain '{normalized}' maxDownloadRequestsPerHour must be greater than or equal to zero.");
|
||||
}
|
||||
}
|
||||
|
||||
if (mirror.Enabled && mirror.Domains.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException("Mirror distribution requires at least one domain when enabled.");
|
||||
}
|
||||
}
|
||||
}
|
||||
1312
src/Concelier/StellaOps.Concelier.WebService/Program.cs
Normal file
1312
src/Concelier/StellaOps.Concelier.WebService/Program.cs
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"profiles": {
|
||||
"StellaOps.Concelier.WebService": {
|
||||
"commandName": "Project",
|
||||
"launchBrowser": true,
|
||||
"environmentVariables": {
|
||||
"ASPNETCORE_ENVIRONMENT": "Development"
|
||||
},
|
||||
"applicationUrl": "https://localhost:50411;http://localhost:50412"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
using System;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.WebService.Options;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Services;
|
||||
|
||||
internal sealed class MirrorFileLocator
|
||||
{
|
||||
private readonly IOptionsMonitor<ConcelierOptions> _options;
|
||||
private readonly ILogger<MirrorFileLocator> _logger;
|
||||
|
||||
public MirrorFileLocator(IOptionsMonitor<ConcelierOptions> options, ILogger<MirrorFileLocator> logger)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public bool TryResolveIndex([NotNullWhen(true)] out string? path, [NotNullWhen(true)] out string? exportId)
|
||||
=> TryResolveRelativePath("index.json", out path, out exportId, out _);
|
||||
|
||||
public bool TryResolveRelativePath(string relativePath, [NotNullWhen(true)] out string? fullPath, [NotNullWhen(true)] out string? exportId, out string? domainId)
|
||||
{
|
||||
fullPath = null;
|
||||
exportId = null;
|
||||
domainId = null;
|
||||
|
||||
var mirror = _options.CurrentValue.Mirror ?? new ConcelierOptions.MirrorOptions();
|
||||
if (!mirror.Enabled)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!TryResolveExportDirectory(mirror, out var exportDirectory, out exportId))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var sanitized = SanitizeRelativePath(relativePath);
|
||||
if (sanitized.Length == 0 || string.Equals(sanitized, "index.json", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
sanitized = $"{mirror.MirrorDirectoryName}/index.json";
|
||||
}
|
||||
|
||||
if (!sanitized.StartsWith($"{mirror.MirrorDirectoryName}/", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var candidate = Combine(exportDirectory, sanitized);
|
||||
if (!CandidateWithinExport(exportDirectory, candidate))
|
||||
{
|
||||
_logger.LogWarning("Rejected mirror export request for path '{RelativePath}' due to traversal attempt.", relativePath);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!File.Exists(candidate))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Extract domain id from path mirror/<domain>/...
|
||||
var segments = sanitized.Split('/', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (segments.Length >= 2)
|
||||
{
|
||||
domainId = segments[1];
|
||||
}
|
||||
|
||||
fullPath = candidate;
|
||||
return true;
|
||||
}
|
||||
|
||||
private bool TryResolveExportDirectory(ConcelierOptions.MirrorOptions mirror, [NotNullWhen(true)] out string? exportDirectory, [NotNullWhen(true)] out string? exportId)
|
||||
{
|
||||
exportDirectory = null;
|
||||
exportId = null;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(mirror.ExportRootAbsolute))
|
||||
{
|
||||
_logger.LogWarning("Mirror export root is not configured; unable to serve mirror content.");
|
||||
return false;
|
||||
}
|
||||
|
||||
var root = mirror.ExportRootAbsolute;
|
||||
var candidateSegment = string.IsNullOrWhiteSpace(mirror.ActiveExportId)
|
||||
? mirror.LatestDirectoryName
|
||||
: mirror.ActiveExportId!;
|
||||
|
||||
if (TryResolveCandidate(root, candidateSegment, mirror.MirrorDirectoryName, out exportDirectory, out exportId))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!string.Equals(candidateSegment, mirror.LatestDirectoryName, StringComparison.OrdinalIgnoreCase)
|
||||
&& TryResolveCandidate(root, mirror.LatestDirectoryName, mirror.MirrorDirectoryName, out exportDirectory, out exportId))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var directories = Directory.Exists(root)
|
||||
? Directory.GetDirectories(root)
|
||||
: Array.Empty<string>();
|
||||
|
||||
Array.Sort(directories, StringComparer.Ordinal);
|
||||
Array.Reverse(directories);
|
||||
|
||||
foreach (var directory in directories)
|
||||
{
|
||||
if (TryResolveCandidate(root, Path.GetFileName(directory), mirror.MirrorDirectoryName, out exportDirectory, out exportId))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to enumerate export directories under {Root}.", root);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private bool TryResolveCandidate(string root, string segment, string mirrorDirectory, [NotNullWhen(true)] out string? exportDirectory, [NotNullWhen(true)] out string? exportId)
|
||||
{
|
||||
exportDirectory = null;
|
||||
exportId = null;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(segment))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var candidate = Path.Combine(root, segment);
|
||||
if (!Directory.Exists(candidate))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var mirrorPath = Path.Combine(candidate, mirrorDirectory);
|
||||
if (!Directory.Exists(mirrorPath))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
exportDirectory = candidate;
|
||||
exportId = segment;
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string SanitizeRelativePath(string relativePath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(relativePath))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var trimmed = relativePath.Replace('\\', '/').Trim().TrimStart('/');
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
private static string Combine(string root, string relativePath)
|
||||
{
|
||||
var segments = relativePath.Split('/', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (segments.Length == 0)
|
||||
{
|
||||
return Path.GetFullPath(root);
|
||||
}
|
||||
|
||||
var combinedRelative = Path.Combine(segments);
|
||||
return Path.GetFullPath(Path.Combine(root, combinedRelative));
|
||||
}
|
||||
|
||||
private static bool CandidateWithinExport(string exportDirectory, string candidate)
|
||||
{
|
||||
var exportRoot = Path.GetFullPath(exportDirectory).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
|
||||
var candidatePath = Path.GetFullPath(candidate);
|
||||
return candidatePath.StartsWith(exportRoot, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Services;
|
||||
|
||||
internal sealed class MirrorRateLimiter
|
||||
{
|
||||
private readonly IMemoryCache _cache;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private static readonly TimeSpan Window = TimeSpan.FromHours(1);
|
||||
|
||||
public MirrorRateLimiter(IMemoryCache cache, TimeProvider timeProvider)
|
||||
{
|
||||
_cache = cache ?? throw new ArgumentNullException(nameof(cache));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
public bool TryAcquire(string domainId, string scope, int limit, out TimeSpan? retryAfter)
|
||||
{
|
||||
retryAfter = null;
|
||||
|
||||
if (limit <= 0 || limit == int.MaxValue)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var key = CreateKey(domainId, scope);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
var counter = _cache.Get<Counter>(key);
|
||||
if (counter is null || now - counter.WindowStart >= Window)
|
||||
{
|
||||
counter = new Counter(now, 0);
|
||||
}
|
||||
|
||||
if (counter.Count >= limit)
|
||||
{
|
||||
var windowEnd = counter.WindowStart + Window;
|
||||
retryAfter = windowEnd > now ? windowEnd - now : TimeSpan.Zero;
|
||||
return false;
|
||||
}
|
||||
|
||||
counter = counter with { Count = counter.Count + 1 };
|
||||
var absoluteExpiration = counter.WindowStart + Window;
|
||||
_cache.Set(key, counter, absoluteExpiration);
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string CreateKey(string domainId, string scope)
|
||||
=> string.Create(domainId.Length + scope.Length + 1, (domainId, scope), static (span, state) =>
|
||||
{
|
||||
state.domainId.AsSpan().CopyTo(span);
|
||||
span[state.domainId.Length] = '|';
|
||||
state.scope.AsSpan().CopyTo(span[(state.domainId.Length + 1)..]);
|
||||
});
|
||||
|
||||
private sealed record Counter(DateTimeOffset WindowStart, int Count);
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<RootNamespace>StellaOps.Concelier.WebService</RootNamespace>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.12.0" />
|
||||
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.12.0" />
|
||||
<PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.12.0" />
|
||||
<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.12.0" />
|
||||
<PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.12.0" />
|
||||
<PackageReference Include="OpenTelemetry.Instrumentation.Process" Version="1.12.0-beta.1" />
|
||||
<PackageReference Include="OpenTelemetry.Instrumentation.Runtime" Version="1.12.0" />
|
||||
<PackageReference Include="Serilog.AspNetCore" Version="8.0.1" />
|
||||
<PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
|
||||
<PackageReference Include="YamlDotNet" Version="13.7.1" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Concelier.Merge/StellaOps.Concelier.Merge.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Configuration/StellaOps.Configuration.csproj" />
|
||||
<ProjectReference Include="../../Authority/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOps.Auth.Abstractions.csproj" />
|
||||
<ProjectReference Include="../../Authority/StellaOps.Authority/StellaOps.Auth.Client/StellaOps.Auth.Client.csproj" />
|
||||
<ProjectReference Include="../../Authority/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOps.Auth.ServerIntegration.csproj" />
|
||||
<ProjectReference Include="../../Aoc/__Libraries/StellaOps.Aoc/StellaOps.Aoc.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
95
src/Concelier/StellaOps.Concelier.WebService/TASKS.md
Normal file
95
src/Concelier/StellaOps.Concelier.WebService/TASKS.md
Normal file
@@ -0,0 +1,95 @@
|
||||
# TASKS — Epic 1: Aggregation-Only Contract
|
||||
> **AOC Reminder:** service links and exposes raw data only—no precedence, severity, or hint computation inside Concelier APIs.
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|---|---|---|---|---|
|
||||
| CONCELIER-WEB-AOC-19-001 `Raw ingestion endpoints` | DONE (2025-10-28) | Concelier WebService Guild | CONCELIER-CORE-AOC-19-001, CONCELIER-STORE-AOC-19-001 | Implement `POST /ingest/advisory`, `GET /advisories/raw*`, and `POST /aoc/verify` minimal API endpoints. Enforce new Authority scopes, inject tenant claims, and surface `AOCWriteGuard` to repository calls. |
|
||||
> Docs alignment (2025-10-26): Endpoint expectations + scope requirements detailed in `docs/ingestion/aggregation-only-contract.md` and `docs/security/authority-scopes.md`.
|
||||
> 2025-10-28: Added coverage for pagination, tenancy enforcement, and ingestion/verification metrics; verified guard handling paths end-to-end.
|
||||
| CONCELIER-WEB-AOC-19-002 `AOC observability` | TODO | Concelier WebService Guild, Observability Guild | CONCELIER-WEB-AOC-19-001 | Emit `ingestion_write_total`, `aoc_violation_total`, latency histograms, and tracing spans (`ingest.fetch/transform/write`, `aoc.guard`). Wire structured logging to include tenant, source vendor, upstream id, and content hash. |
|
||||
> Docs alignment (2025-10-26): Metrics/traces/log schema in `docs/observability/observability.md`.
|
||||
| CONCELIER-WEB-AOC-19-003 `Schema/guard unit tests` | TODO | QA Guild | CONCELIER-WEB-AOC-19-001 | Add unit tests covering schema validation failures, forbidden field rejections (`ERR_AOC_001/002/006/007`), idempotent upserts, and supersedes chains using deterministic fixtures. |
|
||||
> Docs alignment (2025-10-26): Guard rules + error codes documented in AOC reference §5 and CLI guide.
|
||||
| CONCELIER-WEB-AOC-19-004 `End-to-end ingest verification` | TODO | Concelier WebService Guild, QA Guild | CONCELIER-WEB-AOC-19-003, CONCELIER-CORE-AOC-19-002 | Create integration tests ingesting large advisory batches (cold/warm) validating linkset enrichment, metrics emission, and reproducible outputs. Capture load-test scripts + doc notes for Offline Kit dry runs. |
|
||||
> Docs alignment (2025-10-26): Offline verification workflow referenced in `docs/deploy/containers.md` §5.
|
||||
|
||||
## Policy Engine v2
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| CONCELIER-POLICY-20-001 `Policy selection endpoints` | TODO | Concelier WebService Guild | WEB-POLICY-20-001, CONCELIER-CORE-AOC-19-004 | Add batch advisory lookup APIs (`/policy/select/advisories`, `/policy/select/vex`) optimized for PURL/ID lists with pagination, tenant scoping, and explain metadata. |
|
||||
|
||||
## StellaOps Console (Sprint 23)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| CONCELIER-CONSOLE-23-001 `Advisory aggregation views` | TODO | Concelier WebService Guild, BE-Base Platform Guild | CONCELIER-LNM-21-201, CONCELIER-LNM-21-202 | Expose `/console/advisories` endpoints returning aggregation groups (per linkset) with source chips, severity summaries, and provenance metadata for Console list + dashboard cards. Support filters by source, ecosystem, published/modified window, tenant enforcement. |
|
||||
| CONCELIER-CONSOLE-23-002 `Dashboard deltas API` | TODO | Concelier WebService Guild | CONCELIER-CONSOLE-23-001, CONCELIER-LNM-21-203 | Provide aggregated advisory delta counts (new, modified, conflicting) for Console dashboard + live status ticker; emit structured events for queue lag metrics. Ensure deterministic counts across repeated queries. |
|
||||
| CONCELIER-CONSOLE-23-003 `Search fan-out helpers` | TODO | Concelier WebService Guild | CONCELIER-CONSOLE-23-001 | Deliver fast lookup endpoints for CVE/GHSA/purl search (linksets, observations) returning evidence fragments for Console global search; implement caching + scope guards. |
|
||||
|
||||
## Graph Explorer v1
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
|
||||
## Link-Not-Merge v1
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| CONCELIER-LNM-21-201 `Observation APIs` | TODO | Concelier WebService Guild, BE-Base Platform Guild | CONCELIER-LNM-21-001 | Add REST endpoints for advisory observations (`GET /advisories/observations`) with filters (alias, purl, source), pagination, and tenancy enforcement. |
|
||||
| CONCELIER-LNM-21-202 `Linkset APIs` | TODO | Concelier WebService Guild | CONCELIER-LNM-21-002, CONCELIER-LNM-21-003 | Implement linkset read/export endpoints (`/advisories/linksets/{id}`, `/advisories/by-purl/{purl}`, `/advisories/linksets/{id}/export`, `/evidence`) with correlation/conflict payloads and `ERR_AGG_*` mapping. |
|
||||
| CONCELIER-LNM-21-203 `Ingest events` | TODO | Concelier WebService Guild, Platform Events Guild | CONCELIER-LNM-21-005 | Publish NATS/Redis events for new observations/linksets and ensure idempotent consumer contracts; document event schemas. |
|
||||
|
||||
## Graph & Vuln Explorer v1
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| CONCELIER-GRAPH-24-101 `Advisory summary API` | TODO | Concelier WebService Guild | CONCELIER-GRAPH-24-001 | Expose `/advisories/summary` returning raw linkset/observation metadata for overlay services; no derived severity or fix hints. |
|
||||
| CONCELIER-GRAPH-28-102 `Evidence batch API` | TODO | Concelier WebService Guild | CONCELIER-LNM-21-201 | Add batch fetch for advisory observations/linksets keyed by component sets to feed Graph overlay tooltips efficiently. |
|
||||
|
||||
## VEX Lens (Sprint 30)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| CONCELIER-VEXLENS-30-001 `Advisory rationale bridges` | TODO | Concelier WebService Guild, VEX Lens Guild | CONCELIER-VULN-29-001, VEXLENS-30-005 | Guarantee advisory key consistency and cross-links for consensus rationale; Label: VEX-Lens. |
|
||||
|
||||
## Vulnerability Explorer (Sprint 29)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| CONCELIER-VULN-29-001 `Advisory key canonicalization` | TODO | Concelier WebService Guild, Data Integrity Guild | CONCELIER-LNM-21-001 | Canonicalize (lossless) advisory identifiers (CVE/GHSA/vendor) into `advisory_key`, persist `links[]`, expose raw payload snapshots for Explorer evidence tabs; AOC-compliant: no merge, no derived fields, no suppression. Include migration/backfill scripts. |
|
||||
| CONCELIER-VULN-29-002 `Evidence retrieval API` | TODO | Concelier WebService Guild | CONCELIER-VULN-29-001, VULN-API-29-003 | Provide `/vuln/evidence/advisories/{advisory_key}` returning raw advisory docs with provenance, filtering by tenant and source. |
|
||||
| CONCELIER-VULN-29-004 `Observability enhancements` | TODO | Concelier WebService Guild, Observability Guild | CONCELIER-VULN-29-001 | Instrument metrics/logs for advisory normalization (key collisions, withdrawn flags), emit events consumed by Vuln Explorer resolver. |
|
||||
|
||||
## Advisory AI (Sprint 31)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| CONCELIER-AIAI-31-001 `Paragraph anchors` | TODO | Concelier WebService Guild | CONCELIER-VULN-29-001 | Expose advisory chunk API returning paragraph anchors, section metadata, and token-safe text for Advisory AI retrieval. |
|
||||
| CONCELIER-AIAI-31-002 `Structured fields` | TODO | Concelier WebService Guild | CONCELIER-AIAI-31-001 | Ensure normalized advisories expose workaround/fix/CVSS fields via API; add caching for summary queries. |
|
||||
| CONCELIER-AIAI-31-003 `Advisory AI telemetry` | TODO | Concelier WebService Guild, Observability Guild | CONCELIER-AIAI-31-001 | Emit metrics/logs for chunk requests, cache hits, and guardrail blocks triggered by advisory payloads. |
|
||||
|
||||
## Observability & Forensics (Epic 15)
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| CONCELIER-WEB-OBS-50-001 `Telemetry adoption` | TODO | Concelier WebService Guild | TELEMETRY-OBS-50-001, CONCELIER-OBS-50-001 | Adopt telemetry core in web service host, ensure ingest + read endpoints emit trace/log fields (`tenant_id`, `route`, `decision_effect`), and add correlation IDs to responses. |
|
||||
| CONCELIER-WEB-OBS-51-001 `Observability APIs` | TODO | Concelier WebService Guild | CONCELIER-WEB-OBS-50-001, WEB-OBS-51-001 | Surface ingest health metrics, queue depth, and SLO status via `/obs/concelier/health` endpoint for Console widgets, with caching and tenant partitioning. |
|
||||
| CONCELIER-WEB-OBS-52-001 `Timeline streaming` | TODO | Concelier WebService Guild | CONCELIER-WEB-OBS-50-001, TIMELINE-OBS-52-003 | Provide SSE stream `/obs/concelier/timeline` bridging to Timeline Indexer with paging tokens, guardrails, and audit logging. |
|
||||
| CONCELIER-WEB-OBS-53-001 `Evidence locker integration` | TODO | Concelier WebService Guild, Evidence Locker Guild | CONCELIER-OBS-53-001, EVID-OBS-53-003 | Add `/evidence/advisories/*` routes invoking evidence locker snapshots, verifying tenant scopes (`evidence:read`), and returning signed manifest metadata. |
|
||||
| CONCELIER-WEB-OBS-54-001 `Attestation exposure` | TODO | Concelier WebService Guild | CONCELIER-OBS-54-001, PROV-OBS-54-001 | Provide `/attestations/advisories/*` read APIs surfacing DSSE status, verification summary, and provenance chain for Console/CLI. |
|
||||
| CONCELIER-WEB-OBS-55-001 `Incident mode toggles` | TODO | Concelier WebService Guild, DevOps Guild | CONCELIER-OBS-55-001, WEB-OBS-55-001 | Implement incident mode toggle endpoints, propagate to orchestrator/locker, and document cooldown/backoff semantics. |
|
||||
|
||||
## Air-Gapped Mode (Epic 16)
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| CONCELIER-WEB-AIRGAP-56-001 `Mirror import APIs` | TODO | Concelier WebService Guild | AIRGAP-IMP-58-001, CONCELIER-AIRGAP-56-001 | Extend ingestion endpoints to register mirror bundle sources, expose bundle catalog queries, and block external feed URLs in sealed mode. |
|
||||
| CONCELIER-WEB-AIRGAP-56-002 `Airgap status surfaces` | TODO | Concelier WebService Guild | CONCELIER-AIRGAP-57-002, AIRGAP-CTL-56-002 | Add staleness metadata and bundle provenance to advisory APIs (`/advisories/observations`, `/advisories/linksets`). |
|
||||
| CONCELIER-WEB-AIRGAP-57-001 `Error remediation` | TODO | Concelier WebService Guild, AirGap Policy Guild | AIRGAP-POL-56-001 | Map sealed-mode violations to `AIRGAP_EGRESS_BLOCKED` responses with user guidance. |
|
||||
| CONCELIER-WEB-AIRGAP-58-001 `Import timeline emission` | TODO | Concelier WebService Guild, AirGap Importer Guild | CONCELIER-WEB-AIRGAP-56-001, TIMELINE-OBS-53-001 | Emit timeline events for bundle ingestion operations with bundle ID, scope, and actor metadata. |
|
||||
|
||||
## SDKs & OpenAPI (Epic 17)
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| CONCELIER-WEB-OAS-61-001 `/.well-known/openapi` | TODO | Concelier WebService Guild | OAS-61-001 | Implement discovery endpoint emitting Concelier spec with version metadata and ETag. |
|
||||
| CONCELIER-WEB-OAS-61-002 `Error envelope migration` | TODO | Concelier WebService Guild | APIGOV-61-001 | Ensure all API responses use standardized error envelope; update controllers/tests. |
|
||||
| CONCELIER-WEB-OAS-62-001 `Examples expansion` | TODO | Concelier WebService Guild | CONCELIER-OAS-61-002 | Add curated examples for advisory observations/linksets/conflicts; integrate into dev portal. |
|
||||
| CONCELIER-WEB-OAS-63-001 `Deprecation headers` | TODO | Concelier WebService Guild, API Governance Guild | APIGOV-63-001 | Add Sunset/Deprecation headers for retiring endpoints and update documentation/notifications. |
|
||||
Reference in New Issue
Block a user