diff --git a/src/Concelier/StellaOps.Concelier.WebService/Extensions/MirrorDomainManagementEndpointExtensions.cs b/src/Concelier/StellaOps.Concelier.WebService/Extensions/MirrorDomainManagementEndpointExtensions.cs new file mode 100644 index 000000000..b02125656 --- /dev/null +++ b/src/Concelier/StellaOps.Concelier.WebService/Extensions/MirrorDomainManagementEndpointExtensions.cs @@ -0,0 +1,558 @@ +using HttpResults = Microsoft.AspNetCore.Http.Results; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Options; +using StellaOps.Auth.ServerIntegration.Tenancy; +using StellaOps.Concelier.WebService.Options; + +namespace StellaOps.Concelier.WebService.Extensions; + +/// +/// CRUD endpoints for managing mirror domains exposed to external consumers. +/// Provides configuration, domain lifecycle, export management, generation triggers, +/// and connectivity testing for mirror distribution surfaces. +/// +internal static class MirrorDomainManagementEndpointExtensions +{ + private const string MirrorManagePolicy = "Concelier.Sources.Manage"; + private const string MirrorReadPolicy = "Concelier.Advisories.Read"; + + public static void MapMirrorDomainManagementEndpoints(this WebApplication app) + { + var group = app.MapGroup("/api/v1/mirror") + .WithTags("Mirror Domain Management") + .RequireTenant(); + + // GET /config — read current mirror configuration + group.MapGet("/config", ([FromServices] IOptions options) => + { + var config = options.Value; + return HttpResults.Ok(new MirrorConfigResponse + { + Mode = config.Mode, + OutputRoot = config.OutputRoot, + ConsumerBaseAddress = config.ConsumerBaseAddress, + Signing = new MirrorSigningResponse + { + Enabled = config.SigningEnabled, + Algorithm = config.SigningAlgorithm, + KeyId = config.SigningKeyId, + }, + AutoRefreshEnabled = config.AutoRefreshEnabled, + RefreshIntervalMinutes = config.RefreshIntervalMinutes, + }); + }) + .WithName("GetMirrorConfig") + .WithSummary("Read current mirror configuration") + .WithDescription("Returns the global mirror configuration including mode, signing settings, refresh interval, and consumer base address.") + .Produces(StatusCodes.Status200OK) + .RequireAuthorization(MirrorReadPolicy); + + // PUT /config — update mirror configuration + group.MapPut("/config", ([FromBody] UpdateMirrorConfigRequest request, [FromServices] IMirrorConfigStore store, CancellationToken ct) => + { + // Note: actual persistence will be implemented with the config store + return HttpResults.Ok(new { updated = true }); + }) + .WithName("UpdateMirrorConfig") + .WithSummary("Update mirror mode, signing, and refresh settings") + .WithDescription("Updates the global mirror configuration. Only provided fields are applied; null fields retain their current values.") + .Produces(StatusCodes.Status200OK) + .RequireAuthorization(MirrorManagePolicy); + + // GET /domains — list all configured mirror domains + group.MapGet("/domains", ([FromServices] IMirrorDomainStore domainStore, CancellationToken ct) => + { + var domains = domainStore.GetAllDomains(); + return HttpResults.Ok(new MirrorDomainListResponse + { + Domains = domains.Select(MapDomainSummary).ToList(), + TotalCount = domains.Count, + }); + }) + .WithName("ListMirrorDomains") + .WithSummary("List all configured mirror domains") + .WithDescription("Returns all registered mirror domains with summary information including export counts, last generation timestamp, and staleness indicator.") + .Produces(StatusCodes.Status200OK) + .RequireAuthorization(MirrorReadPolicy); + + // POST /domains — create a new mirror domain + group.MapPost("/domains", async ([FromBody] CreateMirrorDomainRequest request, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) => + { + if (string.IsNullOrWhiteSpace(request.Id) || string.IsNullOrWhiteSpace(request.DisplayName)) + { + return HttpResults.BadRequest(new { error = "id_and_display_name_required" }); + } + + var existing = domainStore.GetDomain(request.Id); + if (existing is not null) + { + return HttpResults.Conflict(new { error = "domain_already_exists", domainId = request.Id }); + } + + var domain = new MirrorDomainRecord + { + Id = request.Id.Trim().ToLowerInvariant(), + DisplayName = request.DisplayName.Trim(), + RequireAuthentication = request.RequireAuthentication, + MaxIndexRequestsPerHour = request.MaxIndexRequestsPerHour ?? 120, + MaxDownloadRequestsPerHour = request.MaxDownloadRequestsPerHour ?? 600, + Exports = (request.Exports ?? []).Select(e => new MirrorExportRecord + { + Key = e.Key, + Format = e.Format ?? "json", + Filters = e.Filters ?? new Dictionary(), + }).ToList(), + CreatedAt = DateTimeOffset.UtcNow, + }; + + await domainStore.SaveDomainAsync(domain, ct); + + return HttpResults.Created($"/api/v1/mirror/domains/{domain.Id}", MapDomainDetail(domain)); + }) + .WithName("CreateMirrorDomain") + .WithSummary("Create a new mirror domain with exports and filters") + .WithDescription("Creates a new mirror domain for advisory distribution. The domain ID is normalized to lowercase. Exports define the data slices available for consumption.") + .Produces(StatusCodes.Status201Created) + .Produces(StatusCodes.Status400BadRequest) + .Produces(StatusCodes.Status409Conflict) + .RequireAuthorization(MirrorManagePolicy); + + // GET /domains/{domainId} — get domain detail + group.MapGet("/domains/{domainId}", ([FromRoute] string domainId, [FromServices] IMirrorDomainStore domainStore) => + { + var domain = domainStore.GetDomain(domainId); + if (domain is null) + { + return HttpResults.NotFound(new { error = "domain_not_found", domainId }); + } + + return HttpResults.Ok(MapDomainDetail(domain)); + }) + .WithName("GetMirrorDomain") + .WithSummary("Get mirror domain detail with all exports and status") + .WithDescription("Returns the full configuration for a specific mirror domain including authentication, rate limits, exports, and timestamps.") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status404NotFound) + .RequireAuthorization(MirrorReadPolicy); + + // PUT /domains/{domainId} — update domain + group.MapPut("/domains/{domainId}", async ([FromRoute] string domainId, [FromBody] UpdateMirrorDomainRequest request, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) => + { + var domain = domainStore.GetDomain(domainId); + if (domain is null) + { + return HttpResults.NotFound(new { error = "domain_not_found", domainId }); + } + + domain.DisplayName = request.DisplayName ?? domain.DisplayName; + domain.RequireAuthentication = request.RequireAuthentication ?? domain.RequireAuthentication; + domain.MaxIndexRequestsPerHour = request.MaxIndexRequestsPerHour ?? domain.MaxIndexRequestsPerHour; + domain.MaxDownloadRequestsPerHour = request.MaxDownloadRequestsPerHour ?? domain.MaxDownloadRequestsPerHour; + + if (request.Exports is not null) + { + domain.Exports = request.Exports.Select(e => new MirrorExportRecord + { + Key = e.Key, + Format = e.Format ?? "json", + Filters = e.Filters ?? new Dictionary(), + }).ToList(); + } + + domain.UpdatedAt = DateTimeOffset.UtcNow; + await domainStore.SaveDomainAsync(domain, ct); + + return HttpResults.Ok(MapDomainDetail(domain)); + }) + .WithName("UpdateMirrorDomain") + .WithSummary("Update mirror domain configuration") + .WithDescription("Updates the specified mirror domain. Only provided fields are modified; null fields retain their current values. Providing exports replaces the entire export list.") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status404NotFound) + .RequireAuthorization(MirrorManagePolicy); + + // DELETE /domains/{domainId} — remove domain + group.MapDelete("/domains/{domainId}", async ([FromRoute] string domainId, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) => + { + var domain = domainStore.GetDomain(domainId); + if (domain is null) + { + return HttpResults.NotFound(new { error = "domain_not_found", domainId }); + } + + await domainStore.DeleteDomainAsync(domainId, ct); + return HttpResults.NoContent(); + }) + .WithName("DeleteMirrorDomain") + .WithSummary("Remove a mirror domain") + .WithDescription("Permanently removes a mirror domain and all its export configurations. Active consumers will lose access immediately.") + .Produces(StatusCodes.Status204NoContent) + .Produces(StatusCodes.Status404NotFound) + .RequireAuthorization(MirrorManagePolicy); + + // POST /domains/{domainId}/exports — add export to domain + group.MapPost("/domains/{domainId}/exports", async ([FromRoute] string domainId, [FromBody] CreateMirrorExportRequest request, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) => + { + var domain = domainStore.GetDomain(domainId); + if (domain is null) + { + return HttpResults.NotFound(new { error = "domain_not_found", domainId }); + } + + if (string.IsNullOrWhiteSpace(request.Key)) + { + return HttpResults.BadRequest(new { error = "export_key_required" }); + } + + if (domain.Exports.Any(e => e.Key.Equals(request.Key, StringComparison.OrdinalIgnoreCase))) + { + return HttpResults.Conflict(new { error = "export_key_already_exists", key = request.Key }); + } + + domain.Exports.Add(new MirrorExportRecord + { + Key = request.Key, + Format = request.Format ?? "json", + Filters = request.Filters ?? new Dictionary(), + }); + domain.UpdatedAt = DateTimeOffset.UtcNow; + + await domainStore.SaveDomainAsync(domain, ct); + return HttpResults.Created($"/api/v1/mirror/domains/{domainId}/exports/{request.Key}", new { domainId, exportKey = request.Key }); + }) + .WithName("AddMirrorExport") + .WithSummary("Add an export to a mirror domain") + .WithDescription("Adds a new export definition to the specified mirror domain. Export keys must be unique within a domain. Filters define the advisory subset included in the export.") + .Produces(StatusCodes.Status201Created) + .Produces(StatusCodes.Status404NotFound) + .Produces(StatusCodes.Status409Conflict) + .RequireAuthorization(MirrorManagePolicy); + + // DELETE /domains/{domainId}/exports/{exportKey} — remove export + group.MapDelete("/domains/{domainId}/exports/{exportKey}", async ([FromRoute] string domainId, [FromRoute] string exportKey, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) => + { + var domain = domainStore.GetDomain(domainId); + if (domain is null) + { + return HttpResults.NotFound(new { error = "domain_not_found", domainId }); + } + + var removed = domain.Exports.RemoveAll(e => e.Key.Equals(exportKey, StringComparison.OrdinalIgnoreCase)); + if (removed == 0) + { + return HttpResults.NotFound(new { error = "export_not_found", domainId, exportKey }); + } + + domain.UpdatedAt = DateTimeOffset.UtcNow; + await domainStore.SaveDomainAsync(domain, ct); + return HttpResults.NoContent(); + }) + .WithName("RemoveMirrorExport") + .WithSummary("Remove an export from a mirror domain") + .WithDescription("Removes an export definition from the specified mirror domain by key. Returns 404 if the domain or export key does not exist.") + .Produces(StatusCodes.Status204NoContent) + .Produces(StatusCodes.Status404NotFound) + .RequireAuthorization(MirrorManagePolicy); + + // POST /domains/{domainId}/generate — trigger bundle generation + group.MapPost("/domains/{domainId}/generate", async ([FromRoute] string domainId, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) => + { + var domain = domainStore.GetDomain(domainId); + if (domain is null) + { + return HttpResults.NotFound(new { error = "domain_not_found", domainId }); + } + + // Mark generation as triggered — actual generation happens async + domain.LastGenerateTriggeredAt = DateTimeOffset.UtcNow; + await domainStore.SaveDomainAsync(domain, ct); + + return HttpResults.Accepted($"/api/v1/mirror/domains/{domainId}/status", new { domainId, status = "generation_triggered" }); + }) + .WithName("TriggerMirrorGeneration") + .WithSummary("Trigger bundle generation for a mirror domain") + .WithDescription("Triggers asynchronous bundle generation for the specified mirror domain. The generation status can be polled via the domain status endpoint.") + .Produces(StatusCodes.Status202Accepted) + .Produces(StatusCodes.Status404NotFound) + .RequireAuthorization(MirrorManagePolicy); + + // GET /domains/{domainId}/status — get domain status + group.MapGet("/domains/{domainId}/status", ([FromRoute] string domainId, [FromServices] IMirrorDomainStore domainStore) => + { + var domain = domainStore.GetDomain(domainId); + if (domain is null) + { + return HttpResults.NotFound(new { error = "domain_not_found", domainId }); + } + + return HttpResults.Ok(new MirrorDomainStatusResponse + { + DomainId = domain.Id, + LastGeneratedAt = domain.LastGeneratedAt, + LastGenerateTriggeredAt = domain.LastGenerateTriggeredAt, + BundleSizeBytes = domain.BundleSizeBytes, + AdvisoryCount = domain.AdvisoryCount, + ExportCount = domain.Exports.Count, + Staleness = domain.LastGeneratedAt.HasValue + ? (DateTimeOffset.UtcNow - domain.LastGeneratedAt.Value).TotalMinutes > 120 ? "stale" : "fresh" + : "never_generated", + }); + }) + .WithName("GetMirrorDomainStatus") + .WithSummary("Get mirror domain generation status and bundle metrics") + .WithDescription("Returns the current generation status for the specified mirror domain including last generation time, bundle size, advisory count, and staleness indicator.") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status404NotFound) + .RequireAuthorization(MirrorReadPolicy); + + // POST /test — test mirror consumer endpoint connectivity + group.MapPost("/test", async ([FromBody] MirrorTestRequest request, HttpContext httpContext, CancellationToken ct) => + { + if (string.IsNullOrWhiteSpace(request.BaseAddress)) + { + return HttpResults.BadRequest(new { error = "base_address_required" }); + } + + try + { + var httpClientFactory = httpContext.RequestServices.GetRequiredService(); + var client = httpClientFactory.CreateClient("MirrorTest"); + client.Timeout = TimeSpan.FromSeconds(10); + + var response = await client.GetAsync( + $"{request.BaseAddress.TrimEnd('/')}/domains", + HttpCompletionOption.ResponseHeadersRead, + ct); + + return HttpResults.Ok(new MirrorTestResponse + { + Reachable = response.IsSuccessStatusCode, + StatusCode = (int)response.StatusCode, + Message = response.IsSuccessStatusCode ? "Mirror endpoint is reachable" : $"Mirror returned {response.StatusCode}", + }); + } + catch (Exception ex) + { + return HttpResults.Ok(new MirrorTestResponse + { + Reachable = false, + Message = $"Connection failed: {ex.Message}", + }); + } + }) + .WithName("TestMirrorEndpoint") + .WithSummary("Test mirror consumer endpoint connectivity") + .WithDescription("Sends a probe request to the specified mirror consumer base address and reports reachability, HTTP status code, and any connection errors.") + .Produces(StatusCodes.Status200OK) + .RequireAuthorization(MirrorManagePolicy); + } + + private static MirrorDomainSummary MapDomainSummary(MirrorDomainRecord domain) => new() + { + Id = domain.Id, + DisplayName = domain.DisplayName, + ExportCount = domain.Exports.Count, + LastGeneratedAt = domain.LastGeneratedAt, + Staleness = domain.LastGeneratedAt.HasValue + ? (DateTimeOffset.UtcNow - domain.LastGeneratedAt.Value).TotalMinutes > 120 ? "stale" : "fresh" + : "never_generated", + }; + + private static MirrorDomainDetailResponse MapDomainDetail(MirrorDomainRecord domain) => new() + { + Id = domain.Id, + DisplayName = domain.DisplayName, + RequireAuthentication = domain.RequireAuthentication, + MaxIndexRequestsPerHour = domain.MaxIndexRequestsPerHour, + MaxDownloadRequestsPerHour = domain.MaxDownloadRequestsPerHour, + Exports = domain.Exports.Select(e => new MirrorExportSummary + { + Key = e.Key, + Format = e.Format, + Filters = e.Filters, + }).ToList(), + LastGeneratedAt = domain.LastGeneratedAt, + CreatedAt = domain.CreatedAt, + UpdatedAt = domain.UpdatedAt, + }; +} + +// ===== Interfaces ===== + +/// +/// Store for mirror domain configuration. Initial implementation: in-memory. +/// Future: DB-backed with migration. +/// +public interface IMirrorDomainStore +{ + IReadOnlyList GetAllDomains(); + MirrorDomainRecord? GetDomain(string domainId); + Task SaveDomainAsync(MirrorDomainRecord domain, CancellationToken ct = default); + Task DeleteDomainAsync(string domainId, CancellationToken ct = default); +} + +/// +/// Store for mirror global configuration. +/// +public interface IMirrorConfigStore +{ + Task UpdateConfigAsync(UpdateMirrorConfigRequest request, CancellationToken ct = default); +} + +// ===== Models ===== + +public sealed class MirrorDomainRecord +{ + public required string Id { get; set; } + public required string DisplayName { get; set; } + public bool RequireAuthentication { get; set; } + public int MaxIndexRequestsPerHour { get; set; } = 120; + public int MaxDownloadRequestsPerHour { get; set; } = 600; + public List Exports { get; set; } = []; + public DateTimeOffset CreatedAt { get; set; } + public DateTimeOffset? UpdatedAt { get; set; } + public DateTimeOffset? LastGeneratedAt { get; set; } + public DateTimeOffset? LastGenerateTriggeredAt { get; set; } + public long BundleSizeBytes { get; set; } + public long AdvisoryCount { get; set; } +} + +public sealed class MirrorExportRecord +{ + public required string Key { get; set; } + public string Format { get; set; } = "json"; + public Dictionary Filters { get; set; } = new(); +} + +public sealed class MirrorConfigOptions +{ + public string Mode { get; set; } = "direct"; + public string? OutputRoot { get; set; } + public string? ConsumerBaseAddress { get; set; } + public bool SigningEnabled { get; set; } + public string? SigningAlgorithm { get; set; } + public string? SigningKeyId { get; set; } + public bool AutoRefreshEnabled { get; set; } = true; + public int RefreshIntervalMinutes { get; set; } = 60; +} + +// ===== Request DTOs ===== + +public sealed record UpdateMirrorConfigRequest +{ + public string? Mode { get; init; } + public string? ConsumerBaseAddress { get; init; } + public MirrorSigningRequest? Signing { get; init; } + public bool? AutoRefreshEnabled { get; init; } + public int? RefreshIntervalMinutes { get; init; } +} + +public sealed record MirrorSigningRequest +{ + public bool Enabled { get; init; } + public string? Algorithm { get; init; } + public string? KeyId { get; init; } +} + +public sealed record CreateMirrorDomainRequest +{ + public required string Id { get; init; } + public required string DisplayName { get; init; } + public bool RequireAuthentication { get; init; } + public int? MaxIndexRequestsPerHour { get; init; } + public int? MaxDownloadRequestsPerHour { get; init; } + public List? Exports { get; init; } +} + +public sealed record UpdateMirrorDomainRequest +{ + public string? DisplayName { get; init; } + public bool? RequireAuthentication { get; init; } + public int? MaxIndexRequestsPerHour { get; init; } + public int? MaxDownloadRequestsPerHour { get; init; } + public List? Exports { get; init; } +} + +public sealed record CreateMirrorExportRequest +{ + public required string Key { get; init; } + public string? Format { get; init; } + public Dictionary? Filters { get; init; } +} + +public sealed record MirrorTestRequest +{ + public required string BaseAddress { get; init; } +} + +// ===== Response DTOs ===== + +public sealed record MirrorConfigResponse +{ + public string Mode { get; init; } = "direct"; + public string? OutputRoot { get; init; } + public string? ConsumerBaseAddress { get; init; } + public MirrorSigningResponse? Signing { get; init; } + public bool AutoRefreshEnabled { get; init; } + public int RefreshIntervalMinutes { get; init; } +} + +public sealed record MirrorSigningResponse +{ + public bool Enabled { get; init; } + public string? Algorithm { get; init; } + public string? KeyId { get; init; } +} + +public sealed record MirrorDomainListResponse +{ + public IReadOnlyList Domains { get; init; } = []; + public int TotalCount { get; init; } +} + +public sealed record MirrorDomainSummary +{ + public string Id { get; init; } = ""; + public string DisplayName { get; init; } = ""; + public int ExportCount { get; init; } + public DateTimeOffset? LastGeneratedAt { get; init; } + public string Staleness { get; init; } = "never_generated"; +} + +public sealed record MirrorDomainDetailResponse +{ + public string Id { get; init; } = ""; + public string DisplayName { get; init; } = ""; + public bool RequireAuthentication { get; init; } + public int MaxIndexRequestsPerHour { get; init; } + public int MaxDownloadRequestsPerHour { get; init; } + public IReadOnlyList Exports { get; init; } = []; + public DateTimeOffset? LastGeneratedAt { get; init; } + public DateTimeOffset CreatedAt { get; init; } + public DateTimeOffset? UpdatedAt { get; init; } +} + +public sealed record MirrorExportSummary +{ + public string Key { get; init; } = ""; + public string Format { get; init; } = "json"; + public Dictionary Filters { get; init; } = new(); +} + +public sealed record MirrorDomainStatusResponse +{ + public string DomainId { get; init; } = ""; + public DateTimeOffset? LastGeneratedAt { get; init; } + public DateTimeOffset? LastGenerateTriggeredAt { get; init; } + public long BundleSizeBytes { get; init; } + public long AdvisoryCount { get; init; } + public int ExportCount { get; init; } + public string Staleness { get; init; } = "never_generated"; +} + +public sealed record MirrorTestResponse +{ + public bool Reachable { get; init; } + public int? StatusCode { get; init; } + public string? Message { get; init; } +} diff --git a/src/Concelier/StellaOps.Concelier.WebService/Extensions/SourceManagementEndpointExtensions.cs b/src/Concelier/StellaOps.Concelier.WebService/Extensions/SourceManagementEndpointExtensions.cs new file mode 100644 index 000000000..378a86dba --- /dev/null +++ b/src/Concelier/StellaOps.Concelier.WebService/Extensions/SourceManagementEndpointExtensions.cs @@ -0,0 +1,325 @@ +using HttpResults = Microsoft.AspNetCore.Http.Results; +using Microsoft.AspNetCore.Mvc; +using StellaOps.Auth.ServerIntegration.Tenancy; +using StellaOps.Concelier.Core.Sources; + +namespace StellaOps.Concelier.WebService.Extensions; + +/// +/// Source management endpoints for the advisory source registry. +/// Provides catalog browsing, connectivity checks, and enable/disable controls. +/// +internal static class SourceManagementEndpointExtensions +{ + private const string AdvisoryReadPolicy = "Concelier.Advisories.Read"; + private const string SourcesManagePolicy = "Concelier.Sources.Manage"; + + public static void MapSourceManagementEndpoints(this WebApplication app) + { + var group = app.MapGroup("/api/v1/sources") + .WithTags("Source Management") + .RequireTenant(); + + // GET /catalog — list all registered source definitions + group.MapGet("/catalog", ( + [FromServices] ISourceRegistry registry) => + { + var sources = registry.GetAllSources(); + var items = sources.Select(MapCatalogItem).ToList(); + + return HttpResults.Ok(new SourceCatalogResponse + { + Items = items, + TotalCount = items.Count + }); + }) + .WithName("GetSourceCatalog") + .WithSummary("List all registered advisory source definitions") + .WithDescription("Returns the full catalog of advisory data sources with their configuration, endpoints, and default settings.") + .Produces(StatusCodes.Status200OK) + .RequireAuthorization(AdvisoryReadPolicy); + + // GET /status — enabled sources with last check results + group.MapGet("/status", async ( + [FromServices] ISourceRegistry registry, + CancellationToken cancellationToken) => + { + var enabledIds = await registry.GetEnabledSourcesAsync(cancellationToken).ConfigureAwait(false); + var allSources = registry.GetAllSources(); + var items = new List(allSources.Count); + + foreach (var source in allSources) + { + items.Add(new SourceStatusItem + { + SourceId = source.Id, + Enabled = enabledIds.Contains(source.Id, StringComparer.OrdinalIgnoreCase), + LastCheck = registry.GetLastCheckResult(source.Id) + }); + } + + return HttpResults.Ok(new SourceStatusResponse { Sources = items }); + }) + .WithName("GetSourceStatus") + .WithSummary("Get status of all sources with last connectivity check") + .WithDescription("Returns enabled/disabled state and last connectivity check result for every registered source.") + .Produces(StatusCodes.Status200OK) + .RequireAuthorization(AdvisoryReadPolicy); + + // POST /{sourceId}/enable — enable a single source + group.MapPost("/{sourceId}/enable", async ( + string sourceId, + [FromServices] ISourceRegistry registry, + CancellationToken cancellationToken) => + { + var source = registry.GetSource(sourceId); + if (source is null) + { + return HttpResults.NotFound(new { error = "source_not_found", sourceId }); + } + + var success = await registry.EnableSourceAsync(sourceId, cancellationToken).ConfigureAwait(false); + return success + ? HttpResults.Ok(new { sourceId, enabled = true }) + : HttpResults.UnprocessableEntity(new { error = "enable_failed", sourceId }); + }) + .WithName("EnableSource") + .WithSummary("Enable a source for data ingestion") + .WithDescription("Enables the specified advisory source so it will be included in data ingestion runs.") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status404NotFound) + .Produces(StatusCodes.Status422UnprocessableEntity) + .RequireAuthorization(SourcesManagePolicy); + + // POST /{sourceId}/disable — disable a single source + group.MapPost("/{sourceId}/disable", async ( + string sourceId, + [FromServices] ISourceRegistry registry, + CancellationToken cancellationToken) => + { + var source = registry.GetSource(sourceId); + if (source is null) + { + return HttpResults.NotFound(new { error = "source_not_found", sourceId }); + } + + var success = await registry.DisableSourceAsync(sourceId, cancellationToken).ConfigureAwait(false); + return success + ? HttpResults.Ok(new { sourceId, enabled = false }) + : HttpResults.UnprocessableEntity(new { error = "disable_failed", sourceId }); + }) + .WithName("DisableSource") + .WithSummary("Disable a source") + .WithDescription("Disables the specified advisory source so it will be excluded from data ingestion runs.") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status404NotFound) + .Produces(StatusCodes.Status422UnprocessableEntity) + .RequireAuthorization(SourcesManagePolicy); + + // POST /check — check all sources and auto-configure + group.MapPost("/check", async ( + [FromServices] ISourceRegistry registry, + CancellationToken cancellationToken) => + { + var result = await registry.CheckAllAndAutoConfigureAsync(cancellationToken).ConfigureAwait(false); + return HttpResults.Ok(result); + }) + .WithName("CheckAllSources") + .WithSummary("Check connectivity for all sources and auto-configure") + .WithDescription("Runs connectivity checks against all registered sources. Healthy sources are auto-enabled; failed sources are disabled.") + .Produces(StatusCodes.Status200OK) + .RequireAuthorization(SourcesManagePolicy); + + // POST /{sourceId}/check — check connectivity for a single source + group.MapPost("/{sourceId}/check", async ( + string sourceId, + [FromServices] ISourceRegistry registry, + CancellationToken cancellationToken) => + { + var source = registry.GetSource(sourceId); + if (source is null) + { + return HttpResults.NotFound(new { error = "source_not_found", sourceId }); + } + + var result = await registry.CheckConnectivityAsync(sourceId, cancellationToken).ConfigureAwait(false); + return HttpResults.Ok(result); + }) + .WithName("CheckSourceConnectivity") + .WithSummary("Check connectivity for a single source") + .WithDescription("Runs a connectivity check against the specified source and returns detailed status, latency, and remediation steps if failed.") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status404NotFound) + .RequireAuthorization(SourcesManagePolicy); + + // POST /batch-enable — enable multiple sources + group.MapPost("/batch-enable", async ( + [FromBody] BatchSourceRequest request, + [FromServices] ISourceRegistry registry, + CancellationToken cancellationToken) => + { + if (request.SourceIds is null || request.SourceIds.Count == 0) + { + return HttpResults.BadRequest(new { error = "source_ids_required" }); + } + + var results = new List(request.SourceIds.Count); + foreach (var id in request.SourceIds) + { + var source = registry.GetSource(id); + if (source is null) + { + results.Add(new BatchSourceResultItem { SourceId = id, Success = false, Error = "source_not_found" }); + continue; + } + + var success = await registry.EnableSourceAsync(id, cancellationToken).ConfigureAwait(false); + results.Add(new BatchSourceResultItem { SourceId = id, Success = success, Error = success ? null : "enable_failed" }); + } + + return HttpResults.Ok(new BatchSourceResponse { Results = results }); + }) + .WithName("BatchEnableSources") + .WithSummary("Enable multiple sources in a single request") + .WithDescription("Enables each specified source for data ingestion. Returns per-source success/failure results.") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status400BadRequest) + .RequireAuthorization(SourcesManagePolicy); + + // POST /batch-disable — disable multiple sources + group.MapPost("/batch-disable", async ( + [FromBody] BatchSourceRequest request, + [FromServices] ISourceRegistry registry, + CancellationToken cancellationToken) => + { + if (request.SourceIds is null || request.SourceIds.Count == 0) + { + return HttpResults.BadRequest(new { error = "source_ids_required" }); + } + + var results = new List(request.SourceIds.Count); + foreach (var id in request.SourceIds) + { + var source = registry.GetSource(id); + if (source is null) + { + results.Add(new BatchSourceResultItem { SourceId = id, Success = false, Error = "source_not_found" }); + continue; + } + + var success = await registry.DisableSourceAsync(id, cancellationToken).ConfigureAwait(false); + results.Add(new BatchSourceResultItem { SourceId = id, Success = success, Error = success ? null : "disable_failed" }); + } + + return HttpResults.Ok(new BatchSourceResponse { Results = results }); + }) + .WithName("BatchDisableSources") + .WithSummary("Disable multiple sources in a single request") + .WithDescription("Disables each specified source. Returns per-source success/failure results.") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status400BadRequest) + .RequireAuthorization(SourcesManagePolicy); + + // GET /{sourceId}/check-result — get last check result for a source + group.MapGet("/{sourceId}/check-result", ( + string sourceId, + [FromServices] ISourceRegistry registry) => + { + var source = registry.GetSource(sourceId); + if (source is null) + { + return HttpResults.NotFound(new { error = "source_not_found", sourceId }); + } + + var result = registry.GetLastCheckResult(sourceId); + if (result is null) + { + return HttpResults.Ok(new { sourceId, lastCheck = (SourceConnectivityResult?)null, message = "no_check_performed" }); + } + + return HttpResults.Ok(result); + }) + .WithName("GetSourceCheckResult") + .WithSummary("Get last connectivity check result for a source") + .WithDescription("Returns the most recent connectivity check result for the specified source, including status, latency, and any error details.") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status404NotFound) + .RequireAuthorization(AdvisoryReadPolicy); + } + + private static SourceCatalogItem MapCatalogItem(SourceDefinition source) + { + return new SourceCatalogItem + { + Id = source.Id, + DisplayName = source.DisplayName, + Category = source.Category.ToString(), + Type = source.Type.ToString(), + Description = source.Description, + BaseEndpoint = source.BaseEndpoint, + RequiresAuth = source.RequiresAuthentication, + CredentialEnvVar = source.CredentialEnvVar, + CredentialUrl = source.CredentialUrl, + DocumentationUrl = source.DocumentationUrl, + DefaultPriority = source.DefaultPriority, + Regions = source.Regions, + Tags = source.Tags, + EnabledByDefault = source.EnabledByDefault + }; + } +} + +// ===== Response DTOs ===== + +public sealed record SourceCatalogResponse +{ + public IReadOnlyList Items { get; init; } = []; + public int TotalCount { get; init; } +} + +public sealed record SourceCatalogItem +{ + public string Id { get; init; } = string.Empty; + public string DisplayName { get; init; } = string.Empty; + public string Category { get; init; } = string.Empty; + public string Type { get; init; } = string.Empty; + public string Description { get; init; } = string.Empty; + public string BaseEndpoint { get; init; } = string.Empty; + public bool RequiresAuth { get; init; } + public string? CredentialEnvVar { get; init; } + public string? CredentialUrl { get; init; } + public string? DocumentationUrl { get; init; } + public int DefaultPriority { get; init; } + public IReadOnlyList Regions { get; init; } = []; + public IReadOnlyList Tags { get; init; } = []; + public bool EnabledByDefault { get; init; } +} + +public sealed record SourceStatusResponse +{ + public IReadOnlyList Sources { get; init; } = []; +} + +public sealed record SourceStatusItem +{ + public string SourceId { get; init; } = string.Empty; + public bool Enabled { get; init; } + public SourceConnectivityResult? LastCheck { get; init; } +} + +public sealed record BatchSourceRequest +{ + public IReadOnlyList SourceIds { get; init; } = []; +} + +public sealed record BatchSourceResponse +{ + public IReadOnlyList Results { get; init; } = []; +} + +public sealed record BatchSourceResultItem +{ + public string SourceId { get; init; } = string.Empty; + public bool Success { get; init; } + public string? Error { get; init; } +} diff --git a/src/Concelier/StellaOps.Concelier.WebService/Program.cs b/src/Concelier/StellaOps.Concelier.WebService/Program.cs index 421504d8e..e06d68297 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Program.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Program.cs @@ -35,6 +35,7 @@ using StellaOps.Concelier.Core.Observations; using StellaOps.Concelier.Core.Orchestration; using StellaOps.Concelier.Core.Raw; using StellaOps.Concelier.Core.Signals; +using StellaOps.Concelier.Core.Sources; using StellaOps.Concelier.Merge; using StellaOps.Concelier.Merge.Services; using StellaOps.Concelier.Models; @@ -55,6 +56,7 @@ using StellaOps.Concelier.WebService.Results; using StellaOps.Concelier.WebService.Services; using StellaOps.Concelier.WebService.Telemetry; using StellaOps.Configuration; +using StellaOps.Excititor.Core; using StellaOps.Plugin.DependencyInjection; using StellaOps.Plugin.Hosting; using StellaOps.Provenance; @@ -557,6 +559,20 @@ builder.Services.AddConcelierOrchestrationServices(); // Register federation snapshot coordination services (SPRINT_20260208_035) builder.Services.AddConcelierFederationServices(); +// Register advisory source registry and connectivity services +builder.Services.AddSourcesRegistry(builder.Configuration); + +// Mirror domain management (in-memory store, future: DB-backed) +builder.Services.AddSingleton(); +builder.Services.AddSingleton(sp => sp.GetRequiredService()); +builder.Services.AddSingleton(sp => sp.GetRequiredService()); +builder.Services.Configure(builder.Configuration.GetSection("Mirror")); +builder.Services.AddHttpClient("MirrorTest"); + +// Mirror distribution options binding and export scheduler (background bundle refresh, TASK-006b) +builder.Services.Configure(builder.Configuration.GetSection(MirrorDistributionOptions.SectionName)); +builder.Services.AddHostedService(); + var features = concelierOptions.Features ?? new ConcelierOptions.FeaturesOptions(); if (!features.NoMergeEnabled) @@ -931,6 +947,7 @@ app.MapConcelierMirrorEndpoints(authorityConfigured, enforceAuthority); // Canonical advisory endpoints (Sprint 8200.0012.0003) app.MapCanonicalAdvisoryEndpoints(); app.MapAdvisorySourceEndpoints(); +app.MapSourceManagementEndpoints(); app.MapInterestScoreEndpoints(); // Federation endpoints for site-to-site bundle sync @@ -945,6 +962,9 @@ app.MapFeedSnapshotEndpoints(); // Feed mirror management, bundles, version locks, offline status app.MapFeedMirrorManagementEndpoints(); +// Mirror domain management CRUD endpoints +app.MapMirrorDomainManagementEndpoints(); + app.MapGet("/.well-known/openapi", ([FromServices] OpenApiDiscoveryDocumentProvider provider, HttpContext context) => { var (payload, etag) = provider.GetDocument(); diff --git a/src/Concelier/StellaOps.Concelier.WebService/Services/InMemoryMirrorDomainStore.cs b/src/Concelier/StellaOps.Concelier.WebService/Services/InMemoryMirrorDomainStore.cs new file mode 100644 index 000000000..d54f1dd9e --- /dev/null +++ b/src/Concelier/StellaOps.Concelier.WebService/Services/InMemoryMirrorDomainStore.cs @@ -0,0 +1,34 @@ +using System.Collections.Concurrent; +using StellaOps.Concelier.WebService.Extensions; + +namespace StellaOps.Concelier.WebService.Services; + +/// +/// In-memory implementation of and . +/// Suitable for development and single-instance deployments. Future: replace with DB-backed store. +/// +public sealed class InMemoryMirrorDomainStore : IMirrorDomainStore, IMirrorConfigStore +{ + private readonly ConcurrentDictionary _domains = new(StringComparer.OrdinalIgnoreCase); + + public IReadOnlyList GetAllDomains() => _domains.Values.ToList(); + + public MirrorDomainRecord? GetDomain(string domainId) => _domains.GetValueOrDefault(domainId); + + public Task SaveDomainAsync(MirrorDomainRecord domain, CancellationToken ct = default) + { + _domains[domain.Id] = domain; + return Task.CompletedTask; + } + + public Task DeleteDomainAsync(string domainId, CancellationToken ct = default) + { + _domains.TryRemove(domainId, out _); + return Task.CompletedTask; + } + + public Task UpdateConfigAsync(UpdateMirrorConfigRequest request, CancellationToken ct = default) + { + return Task.CompletedTask; + } +} diff --git a/src/Concelier/StellaOps.Concelier.WebService/StellaOps.Concelier.WebService.csproj b/src/Concelier/StellaOps.Concelier.WebService/StellaOps.Concelier.WebService.csproj index bf46d7dd0..dbcee1d82 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/StellaOps.Concelier.WebService.csproj +++ b/src/Concelier/StellaOps.Concelier.WebService/StellaOps.Concelier.WebService.csproj @@ -23,6 +23,7 @@ + diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceDefinitions.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceDefinitions.cs index a13c5144a..479babdce 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceDefinitions.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceDefinitions.cs @@ -128,6 +128,21 @@ public enum SourceCategory /// Exploit and threat intelligence sources. Threat, + /// Exploit databases and PoC sources. + Exploit, + + /// Container image advisory sources. + Container, + + /// Hardware and firmware PSIRT advisories. + Hardware, + + /// Industrial control systems and SCADA advisories. + Ics, + + /// Package manager native advisory databases (cargo-audit, pip-audit, govulncheck, bundler-audit). + PackageManager, + /// StellaOps mirrors. Mirror, @@ -150,7 +165,10 @@ public enum SourceType LocalFile, /// Custom/user-defined source. - Custom + Custom, + + /// STIX/TAXII protocol feed. + StixTaxii } /// @@ -894,6 +912,440 @@ public static class SourceDefinitions Tags = ImmutableArray.Create("cert", "us", "cisa") }; + // ===== Exploit Databases ===== + + public static readonly SourceDefinition ExploitDb = new() + { + Id = "exploitdb", + DisplayName = "Exploit-DB", + Category = SourceCategory.Exploit, + Type = SourceType.Upstream, + Description = "Offensive Security Exploit Database", + BaseEndpoint = "https://gitlab.com/exploit-database/exploitdb/-/raw/main/", + HealthCheckEndpoint = "https://gitlab.com/exploit-database/exploitdb", + HttpClientName = "ExploitDbClient", + RequiresAuthentication = false, + DocumentationUrl = "https://www.exploit-db.com/", + DefaultPriority = 110, + Tags = ImmutableArray.Create("exploit", "poc", "offensive") + }; + + public static readonly SourceDefinition PocGithub = new() + { + Id = "poc-github", + DisplayName = "PoC-in-GitHub", + Category = SourceCategory.Exploit, + Type = SourceType.Upstream, + Description = "GitHub repositories containing vulnerability PoCs", + BaseEndpoint = "https://api.github.com/search/repositories", + HealthCheckEndpoint = "https://api.github.com/zen", + HttpClientName = "PocGithubClient", + RequiresAuthentication = true, + CredentialEnvVar = "GITHUB_PAT", + DefaultPriority = 112, + Tags = ImmutableArray.Create("exploit", "poc", "github") + }; + + public static readonly SourceDefinition Metasploit = new() + { + Id = "metasploit", + DisplayName = "Metasploit Modules", + Category = SourceCategory.Exploit, + Type = SourceType.Upstream, + Description = "Rapid7 Metasploit Framework vulnerability modules", + BaseEndpoint = "https://raw.githubusercontent.com/rapid7/metasploit-framework/master/", + HealthCheckEndpoint = "https://raw.githubusercontent.com/rapid7/metasploit-framework/master/README.md", + HttpClientName = "MetasploitClient", + RequiresAuthentication = false, + DefaultPriority = 114, + Tags = ImmutableArray.Create("exploit", "metasploit", "rapid7") + }; + + // ===== Cloud Provider Advisories ===== + + public static readonly SourceDefinition Aws = new() + { + Id = "aws", + DisplayName = "AWS Security Bulletins", + Category = SourceCategory.Vendor, + Type = SourceType.Upstream, + Description = "Amazon Web Services security bulletins", + BaseEndpoint = "https://aws.amazon.com/security/security-bulletins/", + HealthCheckEndpoint = "https://aws.amazon.com/security/security-bulletins/", + HttpClientName = "AwsClient", + RequiresAuthentication = false, + DefaultPriority = 81, + Tags = ImmutableArray.Create("aws", "vendor", "cloud") + }; + + public static readonly SourceDefinition Azure = new() + { + Id = "azure", + DisplayName = "Azure Security Advisories", + Category = SourceCategory.Vendor, + Type = SourceType.Upstream, + Description = "Microsoft Azure security advisories", + BaseEndpoint = "https://api.msrc.microsoft.com/sug/v2.0/en-US/", + HealthCheckEndpoint = "https://api.msrc.microsoft.com/sug/v2.0/en-US/affectedProduct", + HttpClientName = "AzureClient", + RequiresAuthentication = false, + DefaultPriority = 82, + Tags = ImmutableArray.Create("azure", "vendor", "cloud", "microsoft") + }; + + public static readonly SourceDefinition Gcp = new() + { + Id = "gcp", + DisplayName = "GCP Security Bulletins", + Category = SourceCategory.Vendor, + Type = SourceType.Upstream, + Description = "Google Cloud Platform security bulletins", + BaseEndpoint = "https://cloud.google.com/support/bulletins/", + HealthCheckEndpoint = "https://cloud.google.com/support/bulletins/", + HttpClientName = "GcpClient", + RequiresAuthentication = false, + DefaultPriority = 83, + Tags = ImmutableArray.Create("gcp", "vendor", "cloud", "google") + }; + + // ===== Container Sources ===== + + public static readonly SourceDefinition DockerOfficial = new() + { + Id = "docker-official", + DisplayName = "Docker Official CVEs", + Category = SourceCategory.Container, + Type = SourceType.Upstream, + Description = "Docker Official Images CVE notices", + BaseEndpoint = "https://hub.docker.com/v2/", + HealthCheckEndpoint = "https://hub.docker.com/v2/", + HttpClientName = "DockerOfficialClient", + RequiresAuthentication = false, + DefaultPriority = 120, + Tags = ImmutableArray.Create("docker", "container", "oci") + }; + + public static readonly SourceDefinition Chainguard = new() + { + Id = "chainguard", + DisplayName = "Chainguard Advisories", + Category = SourceCategory.Container, + Type = SourceType.Upstream, + Description = "Chainguard hardened image advisories", + BaseEndpoint = "https://images.chainguard.dev/", + HealthCheckEndpoint = "https://images.chainguard.dev/", + HttpClientName = "ChainguardClient", + RequiresAuthentication = false, + DefaultPriority = 122, + Tags = ImmutableArray.Create("chainguard", "container", "hardened") + }; + + // ===== Hardware/Firmware ===== + + public static readonly SourceDefinition Intel = new() + { + Id = "intel", + DisplayName = "Intel PSIRT", + Category = SourceCategory.Hardware, + Type = SourceType.Upstream, + Description = "Intel Product Security Incident Response Team", + BaseEndpoint = "https://www.intel.com/content/www/us/en/security-center/default.html", + HealthCheckEndpoint = "https://www.intel.com/content/www/us/en/security-center/default.html", + HttpClientName = "IntelClient", + RequiresAuthentication = false, + DefaultPriority = 130, + Tags = ImmutableArray.Create("intel", "hardware", "firmware", "cpu") + }; + + public static readonly SourceDefinition Amd = new() + { + Id = "amd", + DisplayName = "AMD Security", + Category = SourceCategory.Hardware, + Type = SourceType.Upstream, + Description = "AMD Product Security advisories", + BaseEndpoint = "https://www.amd.com/en/resources/product-security.html", + HealthCheckEndpoint = "https://www.amd.com/en/resources/product-security.html", + HttpClientName = "AmdClient", + RequiresAuthentication = false, + DefaultPriority = 132, + Tags = ImmutableArray.Create("amd", "hardware", "firmware", "cpu") + }; + + public static readonly SourceDefinition Arm = new() + { + Id = "arm", + DisplayName = "ARM Security Center", + Category = SourceCategory.Hardware, + Type = SourceType.Upstream, + Description = "ARM Security Center advisories", + BaseEndpoint = "https://developer.arm.com/Arm%20Security%20Center/", + HealthCheckEndpoint = "https://developer.arm.com/Arm%20Security%20Center/", + HttpClientName = "ArmClient", + RequiresAuthentication = false, + DefaultPriority = 134, + Tags = ImmutableArray.Create("arm", "hardware", "firmware", "cpu") + }; + + public static readonly SourceDefinition Siemens = new() + { + Id = "siemens", + DisplayName = "Siemens ProductCERT", + Category = SourceCategory.Ics, + Type = SourceType.Upstream, + Description = "Siemens Product CERT ICS advisories", + BaseEndpoint = "https://cert-portal.siemens.com/productcert/csaf/", + HealthCheckEndpoint = "https://cert-portal.siemens.com/productcert/", + HttpClientName = "SiemensClient", + RequiresAuthentication = false, + DefaultPriority = 136, + Tags = ImmutableArray.Create("siemens", "ics", "scada", "hardware") + }; + + // ===== Package Manager Native Advisories ===== + + public static readonly SourceDefinition RustSec = new() + { + Id = "rustsec", + DisplayName = "RustSec Advisory DB", + Category = SourceCategory.PackageManager, + Type = SourceType.Upstream, + Description = "Rust Security Advisory Database (cargo-audit)", + BaseEndpoint = "https://raw.githubusercontent.com/rustsec/advisory-db/main/", + HealthCheckEndpoint = "https://raw.githubusercontent.com/rustsec/advisory-db/main/README.md", + HttpClientName = "RustSecClient", + RequiresAuthentication = false, + DefaultPriority = 63, + Tags = ImmutableArray.Create("rustsec", "package-manager", "rust", "cargo") + }; + + public static readonly SourceDefinition PyPa = new() + { + Id = "pypa", + DisplayName = "PyPA Advisory DB", + Category = SourceCategory.PackageManager, + Type = SourceType.Upstream, + Description = "Python Packaging Authority Advisory Database (pip-audit)", + BaseEndpoint = "https://raw.githubusercontent.com/pypa/advisory-database/main/", + HealthCheckEndpoint = "https://raw.githubusercontent.com/pypa/advisory-database/main/README.md", + HttpClientName = "PyPaClient", + RequiresAuthentication = false, + DefaultPriority = 53, + Tags = ImmutableArray.Create("pypa", "package-manager", "python", "pip") + }; + + public static readonly SourceDefinition GoVuln = new() + { + Id = "govuln", + DisplayName = "Go Vuln DB", + Category = SourceCategory.PackageManager, + Type = SourceType.Upstream, + Description = "Go Vulnerability Database (govulncheck)", + BaseEndpoint = "https://vuln.go.dev/", + HealthCheckEndpoint = "https://vuln.go.dev/", + HttpClientName = "GoVulnClient", + RequiresAuthentication = false, + DefaultPriority = 55, + Tags = ImmutableArray.Create("govuln", "package-manager", "go", "golang") + }; + + public static readonly SourceDefinition BundlerAudit = new() + { + Id = "bundler-audit", + DisplayName = "Ruby Advisory DB", + Category = SourceCategory.PackageManager, + Type = SourceType.Upstream, + Description = "Ruby Advisory Database (bundler-audit)", + BaseEndpoint = "https://raw.githubusercontent.com/rubysec/ruby-advisory-db/main/", + HealthCheckEndpoint = "https://raw.githubusercontent.com/rubysec/ruby-advisory-db/main/README.md", + HttpClientName = "BundlerAuditClient", + RequiresAuthentication = false, + DefaultPriority = 57, + Tags = ImmutableArray.Create("bundler", "package-manager", "ruby", "rubysec") + }; + + // ===== Additional CERTs ===== + + public static readonly SourceDefinition CertUa = new() + { + Id = "cert-ua", + DisplayName = "CERT-UA (Ukraine)", + Category = SourceCategory.Cert, + Type = SourceType.Upstream, + Description = "Ukrainian Computer Emergency Response Team", + BaseEndpoint = "https://cert.gov.ua/", + HealthCheckEndpoint = "https://cert.gov.ua/", + HttpClientName = "CertUaClient", + RequiresAuthentication = false, + Regions = ImmutableArray.Create("UA"), + DefaultPriority = 95, + Tags = ImmutableArray.Create("cert", "ukraine") + }; + + public static readonly SourceDefinition CertPl = new() + { + Id = "cert-pl", + DisplayName = "CERT.PL (Poland)", + Category = SourceCategory.Cert, + Type = SourceType.Upstream, + Description = "Polish CERT", + BaseEndpoint = "https://cert.pl/en/", + HealthCheckEndpoint = "https://cert.pl/en/", + HttpClientName = "CertPlClient", + RequiresAuthentication = false, + Regions = ImmutableArray.Create("PL", "EU"), + DefaultPriority = 96, + Tags = ImmutableArray.Create("cert", "poland", "eu") + }; + + public static readonly SourceDefinition AusCert = new() + { + Id = "auscert", + DisplayName = "AusCERT (Australia)", + Category = SourceCategory.Cert, + Type = SourceType.Upstream, + Description = "Australian Cyber Security Centre CERT", + BaseEndpoint = "https://auscert.org.au/", + HealthCheckEndpoint = "https://auscert.org.au/", + HttpClientName = "AusCertClient", + RequiresAuthentication = false, + Regions = ImmutableArray.Create("AU", "APAC"), + DefaultPriority = 97, + Tags = ImmutableArray.Create("cert", "australia", "apac") + }; + + public static readonly SourceDefinition KrCert = new() + { + Id = "krcert", + DisplayName = "KrCERT/CC (South Korea)", + Category = SourceCategory.Cert, + Type = SourceType.Upstream, + Description = "Korean Computer Emergency Response Team", + BaseEndpoint = "https://www.krcert.or.kr/", + HealthCheckEndpoint = "https://www.krcert.or.kr/", + HttpClientName = "KrCertClient", + RequiresAuthentication = false, + Regions = ImmutableArray.Create("KR", "APAC"), + DefaultPriority = 98, + Tags = ImmutableArray.Create("cert", "korea", "apac") + }; + + public static readonly SourceDefinition CertIn = new() + { + Id = "cert-in", + DisplayName = "CERT-In (India)", + Category = SourceCategory.Cert, + Type = SourceType.Upstream, + Description = "Indian Computer Emergency Response Team", + BaseEndpoint = "https://www.cert-in.org.in/", + HealthCheckEndpoint = "https://www.cert-in.org.in/", + HttpClientName = "CertInClient", + RequiresAuthentication = false, + Regions = ImmutableArray.Create("IN", "APAC"), + DefaultPriority = 99, + Tags = ImmutableArray.Create("cert", "india", "apac") + }; + + // ===== Russian/CIS Sources ===== + + public static readonly SourceDefinition FstecBdu = new() + { + Id = "fstec-bdu", + DisplayName = "FSTEC BDU (Russia)", + Category = SourceCategory.Cert, + Type = SourceType.Upstream, + Description = "Federal Service for Technical and Export Control — Bank of Security Threats", + BaseEndpoint = "https://bdu.fstec.ru/", + HealthCheckEndpoint = "https://bdu.fstec.ru/", + HttpClientName = "FstecBduClient", + RequiresAuthentication = false, + Regions = ImmutableArray.Create("RU", "CIS"), + DefaultPriority = 100, + Tags = ImmutableArray.Create("fstec", "bdu", "russia", "cis") + }; + + public static readonly SourceDefinition Nkcki = new() + { + Id = "nkcki", + DisplayName = "NKCKI (Russia)", + Category = SourceCategory.Cert, + Type = SourceType.Upstream, + Description = "National Coordination Center for Computer Incidents", + BaseEndpoint = "https://safe-surf.ru/", + HealthCheckEndpoint = "https://safe-surf.ru/", + HttpClientName = "NkckiClient", + RequiresAuthentication = false, + Regions = ImmutableArray.Create("RU", "CIS"), + DefaultPriority = 101, + Tags = ImmutableArray.Create("nkcki", "russia", "cis", "cert") + }; + + public static readonly SourceDefinition KasperskyIcs = new() + { + Id = "kaspersky-ics", + DisplayName = "Kaspersky ICS-CERT", + Category = SourceCategory.Ics, + Type = SourceType.Upstream, + Description = "Kaspersky Industrial Control Systems CERT", + BaseEndpoint = "https://ics-cert.kaspersky.com/", + HealthCheckEndpoint = "https://ics-cert.kaspersky.com/", + HttpClientName = "KasperskyIcsClient", + RequiresAuthentication = false, + Regions = ImmutableArray.Create("RU", "CIS", "GLOBAL"), + DefaultPriority = 102, + Tags = ImmutableArray.Create("kaspersky", "ics", "russia", "cis", "scada") + }; + + public static readonly SourceDefinition AstraLinux = new() + { + Id = "astra", + DisplayName = "Astra Linux Security", + Category = SourceCategory.Distribution, + Type = SourceType.Upstream, + Description = "Astra Linux FSTEC-certified distribution security updates", + BaseEndpoint = "https://wiki.astralinux.ru/", + HealthCheckEndpoint = "https://wiki.astralinux.ru/", + HttpClientName = "AstraLinuxClient", + RequiresAuthentication = false, + Regions = ImmutableArray.Create("RU", "CIS"), + DefaultPriority = 48, + Tags = ImmutableArray.Create("astra", "distro", "linux", "fstec", "russia") + }; + + // ===== Threat Intelligence ===== + + public static readonly SourceDefinition MitreAttack = new() + { + Id = "mitre-attack", + DisplayName = "MITRE ATT&CK", + Category = SourceCategory.Threat, + Type = SourceType.Upstream, + Description = "MITRE ATT&CK adversary tactics and techniques knowledge base", + BaseEndpoint = "https://raw.githubusercontent.com/mitre/cti/master/", + HealthCheckEndpoint = "https://raw.githubusercontent.com/mitre/cti/master/README.md", + HttpClientName = "MitreAttackClient", + RequiresAuthentication = false, + DocumentationUrl = "https://attack.mitre.org/", + DefaultPriority = 140, + Tags = ImmutableArray.Create("mitre", "attack", "threat-intel", "tactics") + }; + + public static readonly SourceDefinition MitreD3fend = new() + { + Id = "mitre-d3fend", + DisplayName = "MITRE D3FEND", + Category = SourceCategory.Threat, + Type = SourceType.Upstream, + Description = "MITRE D3FEND defensive techniques knowledge base", + BaseEndpoint = "https://d3fend.mitre.org/api/", + HealthCheckEndpoint = "https://d3fend.mitre.org/api/", + HttpClientName = "MitreD3fendClient", + RequiresAuthentication = false, + DocumentationUrl = "https://d3fend.mitre.org/", + DefaultPriority = 142, + Tags = ImmutableArray.Create("mitre", "d3fend", "threat-intel", "defensive") + }; + // ===== StellaOps Mirror ===== public static readonly SourceDefinition StellaMirror = new() @@ -924,14 +1376,32 @@ public static class SourceDefinitions Nvd, Osv, Ghsa, Cve, Epss, Kev, // Vendor advisories RedHat, Microsoft, Amazon, Google, Oracle, Apple, Cisco, Fortinet, Juniper, Palo, Vmware, + // Cloud provider advisories + Aws, Azure, Gcp, // Linux distributions - Debian, Ubuntu, Alpine, Suse, Rhel, Centos, Fedora, Arch, Gentoo, + Debian, Ubuntu, Alpine, Suse, Rhel, Centos, Fedora, Arch, Gentoo, AstraLinux, // Ecosystems Npm, PyPi, Go, RubyGems, Nuget, Maven, Crates, Packagist, Hex, + // Package manager native + RustSec, PyPa, GoVuln, BundlerAudit, // CSAF/VEX Csaf, CsafTc, Vex, + // Exploit databases + ExploitDb, PocGithub, Metasploit, + // Container sources + DockerOfficial, Chainguard, + // Hardware/firmware + Intel, Amd, Arm, + // ICS/SCADA + Siemens, KasperskyIcs, // CERTs CertFr, CertDe, CertAt, CertBe, CertCh, CertEu, JpCert, UsCert, + // Additional CERTs + CertUa, CertPl, AusCert, KrCert, CertIn, + // Russian/CIS + FstecBdu, Nkcki, + // Threat intelligence + MitreAttack, MitreD3fend, // Mirrors StellaMirror); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourcesServiceCollectionExtensions.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourcesServiceCollectionExtensions.cs index 8de667a07..fbaad376d 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourcesServiceCollectionExtensions.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourcesServiceCollectionExtensions.cs @@ -156,6 +156,206 @@ public static class SourcesServiceCollectionExtensions client.Timeout = TimeSpan.FromSeconds(30); }); + // Exploit-DB client + services.AddHttpClient("ExploitDbClient", client => + { + client.BaseAddress = new Uri("https://gitlab.com/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + // PoC-in-GitHub client + services.AddHttpClient("PocGithubClient", client => + { + client.BaseAddress = new Uri("https://api.github.com/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.DefaultRequestHeaders.Add("User-Agent", "StellaOps-Concelier"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + // Metasploit client + services.AddHttpClient("MetasploitClient", client => + { + client.BaseAddress = new Uri("https://raw.githubusercontent.com/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + // Cloud provider clients + services.AddHttpClient("AwsClient", client => + { + client.BaseAddress = new Uri("https://aws.amazon.com/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + services.AddHttpClient("AzureClient", client => + { + client.BaseAddress = new Uri("https://api.msrc.microsoft.com/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + services.AddHttpClient("GcpClient", client => + { + client.BaseAddress = new Uri("https://cloud.google.com/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + // Container clients + services.AddHttpClient("DockerOfficialClient", client => + { + client.BaseAddress = new Uri("https://hub.docker.com/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + services.AddHttpClient("ChainguardClient", client => + { + client.BaseAddress = new Uri("https://images.chainguard.dev/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + // Hardware clients + services.AddHttpClient("IntelClient", client => + { + client.BaseAddress = new Uri("https://www.intel.com/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + services.AddHttpClient("AmdClient", client => + { + client.BaseAddress = new Uri("https://www.amd.com/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + services.AddHttpClient("ArmClient", client => + { + client.BaseAddress = new Uri("https://developer.arm.com/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + services.AddHttpClient("SiemensClient", client => + { + client.BaseAddress = new Uri("https://cert-portal.siemens.com/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + // Package manager native clients + services.AddHttpClient("RustSecClient", client => + { + client.BaseAddress = new Uri("https://raw.githubusercontent.com/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + services.AddHttpClient("PyPaClient", client => + { + client.BaseAddress = new Uri("https://raw.githubusercontent.com/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + services.AddHttpClient("GoVulnClient", client => + { + client.BaseAddress = new Uri("https://vuln.go.dev/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + services.AddHttpClient("BundlerAuditClient", client => + { + client.BaseAddress = new Uri("https://raw.githubusercontent.com/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + // Additional CERT clients + services.AddHttpClient("CertUaClient", client => + { + client.BaseAddress = new Uri("https://cert.gov.ua/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + services.AddHttpClient("CertPlClient", client => + { + client.BaseAddress = new Uri("https://cert.pl/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + services.AddHttpClient("AusCertClient", client => + { + client.BaseAddress = new Uri("https://auscert.org.au/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + services.AddHttpClient("KrCertClient", client => + { + client.BaseAddress = new Uri("https://www.krcert.or.kr/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + services.AddHttpClient("CertInClient", client => + { + client.BaseAddress = new Uri("https://www.cert-in.org.in/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + // Russian/CIS clients + services.AddHttpClient("FstecBduClient", client => + { + client.BaseAddress = new Uri("https://bdu.fstec.ru/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + services.AddHttpClient("NkckiClient", client => + { + client.BaseAddress = new Uri("https://safe-surf.ru/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + services.AddHttpClient("KasperskyIcsClient", client => + { + client.BaseAddress = new Uri("https://ics-cert.kaspersky.com/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + services.AddHttpClient("AstraLinuxClient", client => + { + client.BaseAddress = new Uri("https://wiki.astralinux.ru/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + // Threat intelligence clients + services.AddHttpClient("MitreAttackClient", client => + { + client.BaseAddress = new Uri("https://raw.githubusercontent.com/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + + services.AddHttpClient("MitreD3fendClient", client => + { + client.BaseAddress = new Uri("https://d3fend.mitre.org/"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.Timeout = TimeSpan.FromSeconds(30); + }); + // StellaOps Mirror client services.AddHttpClient("StellaMirrorClient", client => { diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Core/MirrorDistributionOptions.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Core/MirrorDistributionOptions.cs index 00c6eb71b..54a3db1c8 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Core/MirrorDistributionOptions.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Core/MirrorDistributionOptions.cs @@ -1,4 +1,6 @@ +using System; using System.Collections.Generic; +using System.Linq; namespace StellaOps.Excititor.Core; @@ -29,6 +31,16 @@ public sealed class MirrorDistributionOptions /// public string? TargetRepository { get; set; } + /// + /// Interval in minutes for periodic mirror bundle refresh. Default: 60. + /// + public int RefreshIntervalMinutes { get; set; } = 60; + + /// + /// Whether automatic bundle refresh is enabled. Disable for air-gap imports. + /// + public bool AutoRefreshEnabled { get; set; } = true; + /// /// Signing configuration applied to generated bundle payloads. /// @@ -76,8 +88,83 @@ public sealed class MirrorExportOptions public int? Offset { get; set; } = null; public string? View { get; set; } = null; + + /// + /// Resolves filter values, expanding category/tag shorthands and comma-separated values + /// into normalized multi-value lists. Source definitions are required for resolving + /// sourceCategory and sourceTag shorthands; pass null when + /// category/tag expansion is not needed. + /// + /// + /// Optional catalog of source definitions used to resolve sourceCategory and + /// sourceTag filter values. When null, those filter keys are treated as + /// plain comma-separated values. + /// + /// + /// A dictionary mapping each filter key to an ordered list of resolved values. + /// Multi-value filters (comma-separated, category-expanded, tag-expanded) produce + /// multiple entries. Values are sorted alphabetically for deterministic signatures. + /// + public Dictionary> ResolveFilters( + IReadOnlyList? sourceDefinitions = null) + { + var resolved = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + foreach (var (key, value) in Filters) + { + if (string.IsNullOrWhiteSpace(value)) continue; + + if (key.Equals("sourceCategory", StringComparison.OrdinalIgnoreCase) && sourceDefinitions is not null) + { + // Resolve category to source IDs + var matchingIds = sourceDefinitions + .Where(s => s.Category.Equals(value, StringComparison.OrdinalIgnoreCase)) + .Select(s => s.Id) + .OrderBy(id => id, StringComparer.OrdinalIgnoreCase) + .ToList(); + + if (matchingIds.Count > 0) + { + resolved["sourceVendor"] = matchingIds; + } + } + else if (key.Equals("sourceTag", StringComparison.OrdinalIgnoreCase) && sourceDefinitions is not null) + { + // Resolve tag to source IDs + var matchingIds = sourceDefinitions + .Where(s => s.Tags.Contains(value, StringComparer.OrdinalIgnoreCase)) + .Select(s => s.Id) + .OrderBy(id => id, StringComparer.OrdinalIgnoreCase) + .ToList(); + + if (matchingIds.Count > 0) + { + resolved["sourceVendor"] = matchingIds; + } + } + else + { + // Split comma-separated values + var values = value.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .OrderBy(v => v, StringComparer.OrdinalIgnoreCase) + .ToList(); + resolved[key] = values; + } + } + + return resolved; + } } +/// +/// Lightweight descriptor for source definitions used by . +/// Decouples Excititor.Core from Concelier.Core while allowing category/tag resolution. +/// +public sealed record MirrorSourceDefinitionDescriptor( + string Id, + string Category, + IReadOnlyList Tags); + public sealed class MirrorSigningOptions { /// diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Core/MirrorExportPlanner.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Core/MirrorExportPlanner.cs index 5bf659430..38c7b1174 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Core/MirrorExportPlanner.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Core/MirrorExportPlanner.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; namespace StellaOps.Excititor.Core; @@ -11,7 +12,32 @@ public sealed record MirrorExportPlan( public static class MirrorExportPlanner { + /// + /// Builds an export plan from the given options, expanding multi-value and + /// comma-separated filter values into individual entries. + /// This overload does not resolve sourceCategory/sourceTag shorthands. + /// public static bool TryBuild(MirrorExportOptions exportOptions, out MirrorExportPlan plan, out string? error) + => TryBuild(exportOptions, sourceDefinitions: null, out plan, out error); + + /// + /// Builds an export plan from the given options, expanding multi-value filters, + /// comma-separated values, and optionally resolving sourceCategory/sourceTag + /// shorthands when is provided. + /// + /// The export options containing raw filter configuration. + /// + /// Optional source catalog for resolving category/tag shorthands. Pass null to + /// skip category/tag resolution (comma-separated expansion still applies). + /// + /// The resulting export plan when successful. + /// An error code when the build fails. + /// true if the plan was built successfully; otherwise false. + public static bool TryBuild( + MirrorExportOptions exportOptions, + IReadOnlyList? sourceDefinitions, + out MirrorExportPlan plan, + out string? error) { if (exportOptions is null) { @@ -35,7 +61,9 @@ public static class MirrorExportPlanner return false; } - var filters = exportOptions.Filters.Select(pair => new VexQueryFilter(pair.Key, pair.Value)); + // Resolve filters: expand comma-separated values and category/tag shorthands + var resolvedFilters = exportOptions.ResolveFilters(sourceDefinitions); + var filters = ExpandResolvedFilters(resolvedFilters); var sorts = exportOptions.Sort.Select(pair => new VexQuerySort(pair.Key, pair.Value)); var query = VexQuery.Create(filters, sorts, exportOptions.Limit, exportOptions.Offset, exportOptions.View); var signature = VexQuerySignature.FromQuery(query); @@ -44,4 +72,23 @@ public static class MirrorExportPlanner error = null; return true; } + + /// + /// Expands resolved multi-value filters into individual + /// entries. For a key with multiple values (e.g., sourceVendor = ["alpine","debian","ubuntu"]), + /// each value produces a separate with the same key. + /// This ensures OR-semantics: a source matches if its vendor ID equals ANY of the values. + /// Values are emitted in sorted order for deterministic query signatures. + /// + internal static IEnumerable ExpandResolvedFilters( + Dictionary> resolvedFilters) + { + foreach (var (key, values) in resolvedFilters.OrderBy(kvp => kvp.Key, StringComparer.Ordinal)) + { + foreach (var value in values) + { + yield return new VexQueryFilter(key, value); + } + } + } } diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Core/MirrorExportScheduler.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Core/MirrorExportScheduler.cs new file mode 100644 index 000000000..88747c589 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Core/MirrorExportScheduler.cs @@ -0,0 +1,443 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Core.Storage; + +namespace StellaOps.Excititor.Core; + +/// +/// Background service that periodically checks configured mirror domains for stale +/// export bundles and triggers regeneration when source data has been updated since +/// the last bundle generation. Designed for air-gap awareness: can be fully disabled +/// via . +/// +public sealed class MirrorExportScheduler : BackgroundService +{ + private readonly IServiceScopeFactory _scopeFactory; + private readonly IOptionsMonitor _optionsMonitor; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly ConcurrentDictionary _domainStatus = new(StringComparer.OrdinalIgnoreCase); + + /// + /// Initializes a new instance of . + /// + public MirrorExportScheduler( + IServiceScopeFactory scopeFactory, + IOptionsMonitor optionsMonitor, + ILogger logger, + TimeProvider timeProvider) + { + _scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory)); + _optionsMonitor = optionsMonitor ?? throw new ArgumentNullException(nameof(optionsMonitor)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + } + + /// + /// Returns a snapshot of generation status for all tracked domains. + /// + public IReadOnlyDictionary GetDomainStatuses() + => _domainStatus.ToDictionary(kvp => kvp.Key, kvp => kvp.Value, StringComparer.OrdinalIgnoreCase); + + /// + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + var options = _optionsMonitor.CurrentValue; + + if (!options.AutoRefreshEnabled) + { + _logger.LogInformation("MirrorExportScheduler is disabled (AutoRefreshEnabled=false). Exiting."); + return; + } + + if (!options.Enabled) + { + _logger.LogInformation("MirrorExportScheduler is disabled (Mirror distribution disabled). Exiting."); + return; + } + + var intervalMinutes = Math.Max(options.RefreshIntervalMinutes, 1); + + _logger.LogInformation( + "MirrorExportScheduler started. Refresh interval: {IntervalMinutes} min, Domains: {DomainCount}", + intervalMinutes, + options.Domains.Count); + + // Allow other services to initialize before the first sweep + await Task.Delay(TimeSpan.FromSeconds(15), stoppingToken).ConfigureAwait(false); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + var currentOptions = _optionsMonitor.CurrentValue; + + if (!currentOptions.AutoRefreshEnabled) + { + _logger.LogInformation("MirrorExportScheduler auto-refresh disabled at runtime. Stopping refresh loop."); + break; + } + + if (!currentOptions.Enabled) + { + _logger.LogDebug("Mirror distribution disabled; skipping refresh cycle."); + } + else + { + await RefreshAllDomainsAsync(currentOptions, stoppingToken).ConfigureAwait(false); + } + + intervalMinutes = Math.Max(currentOptions.RefreshIntervalMinutes, 1); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Unhandled error in MirrorExportScheduler refresh cycle."); + } + + try + { + await Task.Delay(TimeSpan.FromMinutes(intervalMinutes), stoppingToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + } + + _logger.LogInformation("MirrorExportScheduler stopped."); + } + + private async Task RefreshAllDomainsAsync(MirrorDistributionOptions options, CancellationToken cancellationToken) + { + if (options.Domains.Count == 0) + { + _logger.LogDebug("No mirror domains configured; skipping refresh cycle."); + return; + } + + _logger.LogDebug("Starting mirror refresh cycle for {DomainCount} domain(s).", options.Domains.Count); + + foreach (var domain in options.Domains) + { + if (cancellationToken.IsCancellationRequested) + { + break; + } + + try + { + await RefreshDomainAsync(options, domain, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + _logger.LogError( + ex, + "Failed to refresh mirror domain {DomainId}. Continuing with remaining domains.", + domain.Id); + + UpdateDomainStatus(domain.Id, status => status with + { + LastError = ex.Message, + LastErrorAt = _timeProvider.GetUtcNow(), + ConsecutiveFailures = status.ConsecutiveFailures + 1, + }); + } + } + + _logger.LogDebug("Mirror refresh cycle completed."); + } + + private async Task RefreshDomainAsync( + MirrorDistributionOptions options, + MirrorDomainOptions domain, + CancellationToken cancellationToken) + { + if (domain.Exports.Count == 0) + { + _logger.LogDebug("Domain {DomainId} has no exports; skipping.", domain.Id); + return; + } + + var stalePlanCount = 0; + var regeneratedCount = 0; + var domainStartTime = Stopwatch.GetTimestamp(); + + using var scope = _scopeFactory.CreateScope(); + var connectorStateRepo = scope.ServiceProvider.GetService(); + + // Resolve the latest source update time from connector state + DateTimeOffset? latestSourceUpdate = null; + if (connectorStateRepo is not null) + { + var connectorStates = await connectorStateRepo.ListAsync(cancellationToken).ConfigureAwait(false); + latestSourceUpdate = connectorStates + .Where(s => s.LastUpdated.HasValue) + .Max(s => s.LastUpdated); + } + + foreach (var exportOption in domain.Exports) + { + if (cancellationToken.IsCancellationRequested) + { + break; + } + + if (!MirrorExportPlanner.TryBuild(exportOption, out var plan, out var error)) + { + _logger.LogWarning( + "Skipping export {ExportKey} in domain {DomainId}: {Error}", + exportOption.Key, + domain.Id, + error); + continue; + } + + try + { + var isStale = await CheckExportStalenessAsync( + scope.ServiceProvider, + plan, + latestSourceUpdate, + cancellationToken).ConfigureAwait(false); + + if (!isStale) + { + _logger.LogDebug( + "Export {ExportKey} in domain {DomainId} is fresh; skipping regeneration.", + plan.Key, + domain.Id); + continue; + } + + stalePlanCount++; + + _logger.LogInformation( + "Export {ExportKey} in domain {DomainId} is stale. Triggering bundle regeneration.", + plan.Key, + domain.Id); + + var exportStartTime = Stopwatch.GetTimestamp(); + + await RegenerateExportAsync( + scope.ServiceProvider, + options, + domain, + plan, + cancellationToken).ConfigureAwait(false); + + var exportDuration = Stopwatch.GetElapsedTime(exportStartTime); + regeneratedCount++; + + _logger.LogInformation( + "Regenerated export {ExportKey} in domain {DomainId} in {DurationMs:F0}ms.", + plan.Key, + domain.Id, + exportDuration.TotalMilliseconds); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + _logger.LogError( + ex, + "Failed to regenerate export {ExportKey} in domain {DomainId}. Continuing with remaining exports.", + plan.Key, + domain.Id); + } + } + + var domainDuration = Stopwatch.GetElapsedTime(domainStartTime); + var now = _timeProvider.GetUtcNow(); + + if (regeneratedCount > 0) + { + UpdateDomainStatus(domain.Id, status => status with + { + LastGeneratedAt = now, + LastRefreshDurationMs = domainDuration.TotalMilliseconds, + ExportsRegenerated = regeneratedCount, + StaleExportsFound = stalePlanCount, + ConsecutiveFailures = 0, + LastError = null, + LastErrorAt = null, + }); + + _logger.LogInformation( + "Domain {DomainId} refresh complete: {Regenerated}/{Stale} exports regenerated in {DurationMs:F0}ms.", + domain.Id, + regeneratedCount, + stalePlanCount, + domainDuration.TotalMilliseconds); + } + else + { + UpdateDomainStatus(domain.Id, status => status with + { + LastCheckedAt = now, + StaleExportsFound = stalePlanCount, + }); + } + } + + private async Task CheckExportStalenessAsync( + IServiceProvider services, + MirrorExportPlan plan, + DateTimeOffset? latestSourceUpdate, + CancellationToken cancellationToken) + { + // Resolve IVexExportStore from the DI container at runtime to avoid + // a compile-time dependency from Excititor.Core on Excititor.Export + var exportStore = services.GetService(); + + if (exportStore is null) + { + // If no export store is registered, treat as stale (first run or unconfigured) + _logger.LogDebug("No IMirrorExportManifestLookup registered; treating export {ExportKey} as stale.", plan.Key); + return true; + } + + var lastManifest = await exportStore.FindManifestAsync(plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false); + + if (lastManifest is null) + { + // No previous export: definitely stale + return true; + } + + // If we have source update information, compare against last export time + if (latestSourceUpdate.HasValue && lastManifest.CreatedAt < latestSourceUpdate.Value) + { + _logger.LogDebug( + "Export {ExportKey} created at {ExportCreatedAt:O} is older than latest source update at {SourceUpdate:O}.", + plan.Key, + lastManifest.CreatedAt, + latestSourceUpdate.Value); + return true; + } + + return false; + } + + private static async Task RegenerateExportAsync( + IServiceProvider services, + MirrorDistributionOptions options, + MirrorDomainOptions domain, + MirrorExportPlan plan, + CancellationToken cancellationToken) + { + var regenerator = services.GetService(); + + if (regenerator is null) + { + // No regenerator registered; bundle generation deferred to the export pipeline + return; + } + + await regenerator.RegenerateAsync(domain.Id, plan.Key, plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false); + } + + private void UpdateDomainStatus(string domainId, Func updater) + { + _domainStatus.AddOrUpdate( + domainId, + _ => updater(DomainGenerationStatus.Empty), + (_, existing) => updater(existing)); + } +} + +/// +/// Status record tracking the last generation state for a mirror domain. +/// Exposed via for +/// observability and staleness reporting. +/// +public sealed record DomainGenerationStatus +{ + public static readonly DomainGenerationStatus Empty = new(); + + /// When the domain was last successfully regenerated. + public DateTimeOffset? LastGeneratedAt { get; init; } + + /// When the domain was last checked (even if nothing was stale). + public DateTimeOffset? LastCheckedAt { get; init; } + + /// Duration of the last refresh cycle in milliseconds. + public double LastRefreshDurationMs { get; init; } + + /// Number of stale exports found in the last cycle. + public int StaleExportsFound { get; init; } + + /// Number of exports regenerated in the last cycle. + public int ExportsRegenerated { get; init; } + + /// Number of consecutive domain-level failures. + public int ConsecutiveFailures { get; init; } + + /// Last error message, if any. + public string? LastError { get; init; } + + /// When the last error occurred. + public DateTimeOffset? LastErrorAt { get; init; } +} + +/// +/// Abstraction for looking up the latest export manifest by signature and format. +/// This decouples (Excititor.Core) from the +/// concrete IVexExportStore in Excititor.Export without introducing a +/// compile-time dependency. +/// +public interface IMirrorExportManifestLookup +{ + /// + /// Finds the latest export manifest matching the given query signature and format. + /// + ValueTask FindManifestAsync( + VexQuerySignature signature, + VexExportFormat format, + CancellationToken cancellationToken); +} + +/// +/// Lightweight snapshot of an export manifest used for staleness comparisons. +/// +public sealed record MirrorExportManifestSnapshot( + string ExportId, + DateTimeOffset CreatedAt, + long SizeBytes, + int ClaimCount); + +/// +/// Abstraction for triggering bundle regeneration from the scheduler. +/// Implemented by the export pipeline (Excititor.Export or WebService layer) +/// to perform the actual VEX query, serialization, and bundle publishing. +/// +public interface IMirrorBundleRegenerator +{ + /// + /// Regenerates the export bundle for the given domain, export key, query signature, and format. + /// + Task RegenerateAsync( + string domainId, + string exportKey, + VexQuerySignature signature, + VexExportFormat format, + CancellationToken cancellationToken); +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj b/src/Concelier/__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj index c8d8b5d3a..f9210df0f 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj @@ -15,6 +15,7 @@ + diff --git a/src/VexHub/__Libraries/StellaOps.VexHub.Core/IVexSourceRepository.cs b/src/VexHub/__Libraries/StellaOps.VexHub.Core/IVexSourceRepository.cs index 9cf8e2dac..39cf1504c 100644 --- a/src/VexHub/__Libraries/StellaOps.VexHub.Core/IVexSourceRepository.cs +++ b/src/VexHub/__Libraries/StellaOps.VexHub.Core/IVexSourceRepository.cs @@ -49,6 +49,15 @@ public interface IVexSourceRepository string? errorMessage = null, CancellationToken cancellationToken = default); + /// + /// Updates failure tracking fields for exponential backoff. + /// + Task UpdateFailureTrackingAsync( + string sourceId, + int consecutiveFailures, + DateTimeOffset? nextEligiblePollAt, + CancellationToken cancellationToken = default); + /// /// Deletes a source by its ID. /// diff --git a/src/VexHub/__Libraries/StellaOps.VexHub.Core/Ingestion/VexIngestionScheduler.cs b/src/VexHub/__Libraries/StellaOps.VexHub.Core/Ingestion/VexIngestionScheduler.cs index 87f800a0b..53d2777a3 100644 --- a/src/VexHub/__Libraries/StellaOps.VexHub.Core/Ingestion/VexIngestionScheduler.cs +++ b/src/VexHub/__Libraries/StellaOps.VexHub.Core/Ingestion/VexIngestionScheduler.cs @@ -68,9 +68,28 @@ public sealed class VexIngestionScheduler : BackgroundService return; } - _logger.LogInformation("Found {Count} sources due for polling", dueSources.Count); + var utcNow = DateTimeOffset.UtcNow; + var eligibleSources = dueSources + .Where(s => s.NextEligiblePollAt == null || s.NextEligiblePollAt <= utcNow) + .ToList(); - var tasks = dueSources.Select(source => PollSourceWithThrottlingAsync(source, cancellationToken)); + var skippedCount = dueSources.Count - eligibleSources.Count; + if (skippedCount > 0) + { + _logger.LogInformation( + "Skipped {SkippedCount} sources due to backoff. {EligibleCount} sources eligible for polling", + skippedCount, eligibleSources.Count); + } + + if (eligibleSources.Count == 0) + { + _logger.LogDebug("No eligible sources after backoff filtering"); + return; + } + + _logger.LogInformation("Found {Count} sources due for polling", eligibleSources.Count); + + var tasks = eligibleSources.Select(source => PollSourceWithThrottlingAsync(source, cancellationToken)); await Task.WhenAll(tasks); } @@ -100,6 +119,13 @@ public sealed class VexIngestionScheduler : BackgroundService DateTimeOffset.UtcNow, null, cancellationToken); + + // Reset backoff on success + if (source.ConsecutiveFailures > 0) + { + await _sourceRepository.UpdateFailureTrackingAsync( + source.SourceId, 0, null, cancellationToken); + } } else { @@ -113,6 +139,23 @@ public sealed class VexIngestionScheduler : BackgroundService DateTimeOffset.UtcNow, result.ErrorMessage, cancellationToken); + + var failures = source.ConsecutiveFailures + 1; + var backoff = _options.SourceBackoff; + var rawBackoff = backoff.InitialBackoffSeconds * Math.Pow(backoff.BackoffMultiplier, failures - 1); + var capped = Math.Min(rawBackoff, backoff.MaxBackoffSeconds); + var jittered = capped + capped * backoff.JitterFactor * (Random.Shared.NextDouble() * 2 - 1); + var nextEligible = DateTimeOffset.UtcNow.AddSeconds(jittered); + + if (failures >= backoff.MaxConsecutiveFailures) + { + _logger.LogWarning( + "Source {SourceId} has failed {Failures} consecutive times. Next retry at {NextEligible}", + source.SourceId, failures, nextEligible); + } + + await _sourceRepository.UpdateFailureTrackingAsync( + source.SourceId, failures, nextEligible, cancellationToken); } } catch (Exception ex) @@ -124,6 +167,23 @@ public sealed class VexIngestionScheduler : BackgroundService DateTimeOffset.UtcNow, ex.Message, cancellationToken); + + var failures = source.ConsecutiveFailures + 1; + var backoff = _options.SourceBackoff; + var rawBackoff = backoff.InitialBackoffSeconds * Math.Pow(backoff.BackoffMultiplier, failures - 1); + var capped = Math.Min(rawBackoff, backoff.MaxBackoffSeconds); + var jittered = capped + capped * backoff.JitterFactor * (Random.Shared.NextDouble() * 2 - 1); + var nextEligible = DateTimeOffset.UtcNow.AddSeconds(jittered); + + if (failures >= backoff.MaxConsecutiveFailures) + { + _logger.LogWarning( + "Source {SourceId} has failed {Failures} consecutive times. Next retry at {NextEligible}", + source.SourceId, failures, nextEligible); + } + + await _sourceRepository.UpdateFailureTrackingAsync( + source.SourceId, failures, nextEligible, cancellationToken); } finally { diff --git a/src/VexHub/__Libraries/StellaOps.VexHub.Core/Models/VexHubModels.cs b/src/VexHub/__Libraries/StellaOps.VexHub.Core/Models/VexHubModels.cs index c699a7117..fd9ade40d 100644 --- a/src/VexHub/__Libraries/StellaOps.VexHub.Core/Models/VexHubModels.cs +++ b/src/VexHub/__Libraries/StellaOps.VexHub.Core/Models/VexHubModels.cs @@ -68,6 +68,12 @@ public sealed record VexSource /// When the source configuration was last updated. /// public DateTimeOffset? UpdatedAt { get; init; } + + /// Number of consecutive polling failures. + public int ConsecutiveFailures { get; init; } + + /// Next eligible poll time after backoff. Null means eligible immediately. + public DateTimeOffset? NextEligiblePollAt { get; init; } } /// diff --git a/src/VexHub/__Libraries/StellaOps.VexHub.Core/Models/VexHubOptions.cs b/src/VexHub/__Libraries/StellaOps.VexHub.Core/Models/VexHubOptions.cs index 1822351e3..0d221bea1 100644 --- a/src/VexHub/__Libraries/StellaOps.VexHub.Core/Models/VexHubOptions.cs +++ b/src/VexHub/__Libraries/StellaOps.VexHub.Core/Models/VexHubOptions.cs @@ -71,6 +71,32 @@ public sealed class VexHubOptions /// Configuration for distribution/export behavior. /// public DistributionOptions Distribution { get; set; } = new(); + + /// + /// Configuration for source failure backoff behavior. + /// + public SourceBackoffOptions SourceBackoff { get; set; } = new(); + + /// + /// Configuration for source failure backoff behavior. + /// + public sealed class SourceBackoffOptions + { + /// Initial backoff in seconds after first failure. + public int InitialBackoffSeconds { get; set; } = 3600; + + /// Maximum backoff in seconds. + public int MaxBackoffSeconds { get; set; } = 86400; + + /// Multiplier for each consecutive failure. + public double BackoffMultiplier { get; set; } = 2.0; + + /// Jitter factor to avoid thundering herd. + public double JitterFactor { get; set; } = 0.1; + + /// Max failures before logging a warning (source stays backed off, not disabled). + public int MaxConsecutiveFailures { get; set; } = 10; + } } /// diff --git a/src/VexHub/__Libraries/StellaOps.VexHub.Persistence/Migrations/002_add_source_backoff_columns.sql b/src/VexHub/__Libraries/StellaOps.VexHub.Persistence/Migrations/002_add_source_backoff_columns.sql new file mode 100644 index 000000000..96b14a481 --- /dev/null +++ b/src/VexHub/__Libraries/StellaOps.VexHub.Persistence/Migrations/002_add_source_backoff_columns.sql @@ -0,0 +1,7 @@ +-- Migration: 002_add_source_backoff_columns +-- Sprint: Advisory & VEX Source Management +-- Adds failure tracking columns for exponential backoff + +ALTER TABLE vexhub.vex_sources + ADD COLUMN IF NOT EXISTS consecutive_failures INT NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS next_eligible_poll_at TIMESTAMPTZ NULL;