Expand advisory source catalog to 75 sources and add mirror management backend

Source catalog: add 28 sources across 6 new categories (Exploit, Container,
Hardware, ICS, PackageManager, additional CERTs) plus RU/CIS promotion and
threat intel frameworks. Register 25 new HTTP clients.

Source management API: 9 endpoints under /api/v1/sources for catalog browsing,
connectivity checks, and enable/disable controls.

Mirror domain API: 12 endpoints under /api/v1/mirror for domain CRUD, export
management, on-demand bundle generation, and connectivity testing.

Filter model: multi-value sourceVendor (comma-separated OR), sourceCategory
and sourceTag shorthand resolution via ResolveFilters(). Backward-compatible
with existing single-value filters. Deterministic query signatures.

Mirror export scheduler: BackgroundService with configurable refresh interval,
per-domain staleness detection, error isolation, and air-gap disable toggle.

VEX ingestion backoff: exponential backoff for failed sources (1hr → 24hr cap)
with jitter. New DB migration for tracking columns.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
master
2026-03-15 13:26:52 +02:00
parent 27d27b1952
commit 3931b7e2cf
16 changed files with 2299 additions and 5 deletions

View File

@@ -0,0 +1,558 @@
using HttpResults = Microsoft.AspNetCore.Http.Results;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using StellaOps.Auth.ServerIntegration.Tenancy;
using StellaOps.Concelier.WebService.Options;
namespace StellaOps.Concelier.WebService.Extensions;
/// <summary>
/// CRUD endpoints for managing mirror domains exposed to external consumers.
/// Provides configuration, domain lifecycle, export management, generation triggers,
/// and connectivity testing for mirror distribution surfaces.
/// </summary>
internal static class MirrorDomainManagementEndpointExtensions
{
private const string MirrorManagePolicy = "Concelier.Sources.Manage";
private const string MirrorReadPolicy = "Concelier.Advisories.Read";
public static void MapMirrorDomainManagementEndpoints(this WebApplication app)
{
var group = app.MapGroup("/api/v1/mirror")
.WithTags("Mirror Domain Management")
.RequireTenant();
// GET /config — read current mirror configuration
group.MapGet("/config", ([FromServices] IOptions<MirrorConfigOptions> options) =>
{
var config = options.Value;
return HttpResults.Ok(new MirrorConfigResponse
{
Mode = config.Mode,
OutputRoot = config.OutputRoot,
ConsumerBaseAddress = config.ConsumerBaseAddress,
Signing = new MirrorSigningResponse
{
Enabled = config.SigningEnabled,
Algorithm = config.SigningAlgorithm,
KeyId = config.SigningKeyId,
},
AutoRefreshEnabled = config.AutoRefreshEnabled,
RefreshIntervalMinutes = config.RefreshIntervalMinutes,
});
})
.WithName("GetMirrorConfig")
.WithSummary("Read current mirror configuration")
.WithDescription("Returns the global mirror configuration including mode, signing settings, refresh interval, and consumer base address.")
.Produces<MirrorConfigResponse>(StatusCodes.Status200OK)
.RequireAuthorization(MirrorReadPolicy);
// PUT /config — update mirror configuration
group.MapPut("/config", ([FromBody] UpdateMirrorConfigRequest request, [FromServices] IMirrorConfigStore store, CancellationToken ct) =>
{
// Note: actual persistence will be implemented with the config store
return HttpResults.Ok(new { updated = true });
})
.WithName("UpdateMirrorConfig")
.WithSummary("Update mirror mode, signing, and refresh settings")
.WithDescription("Updates the global mirror configuration. Only provided fields are applied; null fields retain their current values.")
.Produces(StatusCodes.Status200OK)
.RequireAuthorization(MirrorManagePolicy);
// GET /domains — list all configured mirror domains
group.MapGet("/domains", ([FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
{
var domains = domainStore.GetAllDomains();
return HttpResults.Ok(new MirrorDomainListResponse
{
Domains = domains.Select(MapDomainSummary).ToList(),
TotalCount = domains.Count,
});
})
.WithName("ListMirrorDomains")
.WithSummary("List all configured mirror domains")
.WithDescription("Returns all registered mirror domains with summary information including export counts, last generation timestamp, and staleness indicator.")
.Produces<MirrorDomainListResponse>(StatusCodes.Status200OK)
.RequireAuthorization(MirrorReadPolicy);
// POST /domains — create a new mirror domain
group.MapPost("/domains", async ([FromBody] CreateMirrorDomainRequest request, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
{
if (string.IsNullOrWhiteSpace(request.Id) || string.IsNullOrWhiteSpace(request.DisplayName))
{
return HttpResults.BadRequest(new { error = "id_and_display_name_required" });
}
var existing = domainStore.GetDomain(request.Id);
if (existing is not null)
{
return HttpResults.Conflict(new { error = "domain_already_exists", domainId = request.Id });
}
var domain = new MirrorDomainRecord
{
Id = request.Id.Trim().ToLowerInvariant(),
DisplayName = request.DisplayName.Trim(),
RequireAuthentication = request.RequireAuthentication,
MaxIndexRequestsPerHour = request.MaxIndexRequestsPerHour ?? 120,
MaxDownloadRequestsPerHour = request.MaxDownloadRequestsPerHour ?? 600,
Exports = (request.Exports ?? []).Select(e => new MirrorExportRecord
{
Key = e.Key,
Format = e.Format ?? "json",
Filters = e.Filters ?? new Dictionary<string, string>(),
}).ToList(),
CreatedAt = DateTimeOffset.UtcNow,
};
await domainStore.SaveDomainAsync(domain, ct);
return HttpResults.Created($"/api/v1/mirror/domains/{domain.Id}", MapDomainDetail(domain));
})
.WithName("CreateMirrorDomain")
.WithSummary("Create a new mirror domain with exports and filters")
.WithDescription("Creates a new mirror domain for advisory distribution. The domain ID is normalized to lowercase. Exports define the data slices available for consumption.")
.Produces<MirrorDomainDetailResponse>(StatusCodes.Status201Created)
.Produces(StatusCodes.Status400BadRequest)
.Produces(StatusCodes.Status409Conflict)
.RequireAuthorization(MirrorManagePolicy);
// GET /domains/{domainId} — get domain detail
group.MapGet("/domains/{domainId}", ([FromRoute] string domainId, [FromServices] IMirrorDomainStore domainStore) =>
{
var domain = domainStore.GetDomain(domainId);
if (domain is null)
{
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
}
return HttpResults.Ok(MapDomainDetail(domain));
})
.WithName("GetMirrorDomain")
.WithSummary("Get mirror domain detail with all exports and status")
.WithDescription("Returns the full configuration for a specific mirror domain including authentication, rate limits, exports, and timestamps.")
.Produces<MirrorDomainDetailResponse>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(MirrorReadPolicy);
// PUT /domains/{domainId} — update domain
group.MapPut("/domains/{domainId}", async ([FromRoute] string domainId, [FromBody] UpdateMirrorDomainRequest request, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
{
var domain = domainStore.GetDomain(domainId);
if (domain is null)
{
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
}
domain.DisplayName = request.DisplayName ?? domain.DisplayName;
domain.RequireAuthentication = request.RequireAuthentication ?? domain.RequireAuthentication;
domain.MaxIndexRequestsPerHour = request.MaxIndexRequestsPerHour ?? domain.MaxIndexRequestsPerHour;
domain.MaxDownloadRequestsPerHour = request.MaxDownloadRequestsPerHour ?? domain.MaxDownloadRequestsPerHour;
if (request.Exports is not null)
{
domain.Exports = request.Exports.Select(e => new MirrorExportRecord
{
Key = e.Key,
Format = e.Format ?? "json",
Filters = e.Filters ?? new Dictionary<string, string>(),
}).ToList();
}
domain.UpdatedAt = DateTimeOffset.UtcNow;
await domainStore.SaveDomainAsync(domain, ct);
return HttpResults.Ok(MapDomainDetail(domain));
})
.WithName("UpdateMirrorDomain")
.WithSummary("Update mirror domain configuration")
.WithDescription("Updates the specified mirror domain. Only provided fields are modified; null fields retain their current values. Providing exports replaces the entire export list.")
.Produces<MirrorDomainDetailResponse>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(MirrorManagePolicy);
// DELETE /domains/{domainId} — remove domain
group.MapDelete("/domains/{domainId}", async ([FromRoute] string domainId, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
{
var domain = domainStore.GetDomain(domainId);
if (domain is null)
{
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
}
await domainStore.DeleteDomainAsync(domainId, ct);
return HttpResults.NoContent();
})
.WithName("DeleteMirrorDomain")
.WithSummary("Remove a mirror domain")
.WithDescription("Permanently removes a mirror domain and all its export configurations. Active consumers will lose access immediately.")
.Produces(StatusCodes.Status204NoContent)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(MirrorManagePolicy);
// POST /domains/{domainId}/exports — add export to domain
group.MapPost("/domains/{domainId}/exports", async ([FromRoute] string domainId, [FromBody] CreateMirrorExportRequest request, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
{
var domain = domainStore.GetDomain(domainId);
if (domain is null)
{
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
}
if (string.IsNullOrWhiteSpace(request.Key))
{
return HttpResults.BadRequest(new { error = "export_key_required" });
}
if (domain.Exports.Any(e => e.Key.Equals(request.Key, StringComparison.OrdinalIgnoreCase)))
{
return HttpResults.Conflict(new { error = "export_key_already_exists", key = request.Key });
}
domain.Exports.Add(new MirrorExportRecord
{
Key = request.Key,
Format = request.Format ?? "json",
Filters = request.Filters ?? new Dictionary<string, string>(),
});
domain.UpdatedAt = DateTimeOffset.UtcNow;
await domainStore.SaveDomainAsync(domain, ct);
return HttpResults.Created($"/api/v1/mirror/domains/{domainId}/exports/{request.Key}", new { domainId, exportKey = request.Key });
})
.WithName("AddMirrorExport")
.WithSummary("Add an export to a mirror domain")
.WithDescription("Adds a new export definition to the specified mirror domain. Export keys must be unique within a domain. Filters define the advisory subset included in the export.")
.Produces(StatusCodes.Status201Created)
.Produces(StatusCodes.Status404NotFound)
.Produces(StatusCodes.Status409Conflict)
.RequireAuthorization(MirrorManagePolicy);
// DELETE /domains/{domainId}/exports/{exportKey} — remove export
group.MapDelete("/domains/{domainId}/exports/{exportKey}", async ([FromRoute] string domainId, [FromRoute] string exportKey, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
{
var domain = domainStore.GetDomain(domainId);
if (domain is null)
{
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
}
var removed = domain.Exports.RemoveAll(e => e.Key.Equals(exportKey, StringComparison.OrdinalIgnoreCase));
if (removed == 0)
{
return HttpResults.NotFound(new { error = "export_not_found", domainId, exportKey });
}
domain.UpdatedAt = DateTimeOffset.UtcNow;
await domainStore.SaveDomainAsync(domain, ct);
return HttpResults.NoContent();
})
.WithName("RemoveMirrorExport")
.WithSummary("Remove an export from a mirror domain")
.WithDescription("Removes an export definition from the specified mirror domain by key. Returns 404 if the domain or export key does not exist.")
.Produces(StatusCodes.Status204NoContent)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(MirrorManagePolicy);
// POST /domains/{domainId}/generate — trigger bundle generation
group.MapPost("/domains/{domainId}/generate", async ([FromRoute] string domainId, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
{
var domain = domainStore.GetDomain(domainId);
if (domain is null)
{
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
}
// Mark generation as triggered — actual generation happens async
domain.LastGenerateTriggeredAt = DateTimeOffset.UtcNow;
await domainStore.SaveDomainAsync(domain, ct);
return HttpResults.Accepted($"/api/v1/mirror/domains/{domainId}/status", new { domainId, status = "generation_triggered" });
})
.WithName("TriggerMirrorGeneration")
.WithSummary("Trigger bundle generation for a mirror domain")
.WithDescription("Triggers asynchronous bundle generation for the specified mirror domain. The generation status can be polled via the domain status endpoint.")
.Produces(StatusCodes.Status202Accepted)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(MirrorManagePolicy);
// GET /domains/{domainId}/status — get domain status
group.MapGet("/domains/{domainId}/status", ([FromRoute] string domainId, [FromServices] IMirrorDomainStore domainStore) =>
{
var domain = domainStore.GetDomain(domainId);
if (domain is null)
{
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
}
return HttpResults.Ok(new MirrorDomainStatusResponse
{
DomainId = domain.Id,
LastGeneratedAt = domain.LastGeneratedAt,
LastGenerateTriggeredAt = domain.LastGenerateTriggeredAt,
BundleSizeBytes = domain.BundleSizeBytes,
AdvisoryCount = domain.AdvisoryCount,
ExportCount = domain.Exports.Count,
Staleness = domain.LastGeneratedAt.HasValue
? (DateTimeOffset.UtcNow - domain.LastGeneratedAt.Value).TotalMinutes > 120 ? "stale" : "fresh"
: "never_generated",
});
})
.WithName("GetMirrorDomainStatus")
.WithSummary("Get mirror domain generation status and bundle metrics")
.WithDescription("Returns the current generation status for the specified mirror domain including last generation time, bundle size, advisory count, and staleness indicator.")
.Produces<MirrorDomainStatusResponse>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(MirrorReadPolicy);
// POST /test — test mirror consumer endpoint connectivity
group.MapPost("/test", async ([FromBody] MirrorTestRequest request, HttpContext httpContext, CancellationToken ct) =>
{
if (string.IsNullOrWhiteSpace(request.BaseAddress))
{
return HttpResults.BadRequest(new { error = "base_address_required" });
}
try
{
var httpClientFactory = httpContext.RequestServices.GetRequiredService<IHttpClientFactory>();
var client = httpClientFactory.CreateClient("MirrorTest");
client.Timeout = TimeSpan.FromSeconds(10);
var response = await client.GetAsync(
$"{request.BaseAddress.TrimEnd('/')}/domains",
HttpCompletionOption.ResponseHeadersRead,
ct);
return HttpResults.Ok(new MirrorTestResponse
{
Reachable = response.IsSuccessStatusCode,
StatusCode = (int)response.StatusCode,
Message = response.IsSuccessStatusCode ? "Mirror endpoint is reachable" : $"Mirror returned {response.StatusCode}",
});
}
catch (Exception ex)
{
return HttpResults.Ok(new MirrorTestResponse
{
Reachable = false,
Message = $"Connection failed: {ex.Message}",
});
}
})
.WithName("TestMirrorEndpoint")
.WithSummary("Test mirror consumer endpoint connectivity")
.WithDescription("Sends a probe request to the specified mirror consumer base address and reports reachability, HTTP status code, and any connection errors.")
.Produces<MirrorTestResponse>(StatusCodes.Status200OK)
.RequireAuthorization(MirrorManagePolicy);
}
private static MirrorDomainSummary MapDomainSummary(MirrorDomainRecord domain) => new()
{
Id = domain.Id,
DisplayName = domain.DisplayName,
ExportCount = domain.Exports.Count,
LastGeneratedAt = domain.LastGeneratedAt,
Staleness = domain.LastGeneratedAt.HasValue
? (DateTimeOffset.UtcNow - domain.LastGeneratedAt.Value).TotalMinutes > 120 ? "stale" : "fresh"
: "never_generated",
};
private static MirrorDomainDetailResponse MapDomainDetail(MirrorDomainRecord domain) => new()
{
Id = domain.Id,
DisplayName = domain.DisplayName,
RequireAuthentication = domain.RequireAuthentication,
MaxIndexRequestsPerHour = domain.MaxIndexRequestsPerHour,
MaxDownloadRequestsPerHour = domain.MaxDownloadRequestsPerHour,
Exports = domain.Exports.Select(e => new MirrorExportSummary
{
Key = e.Key,
Format = e.Format,
Filters = e.Filters,
}).ToList(),
LastGeneratedAt = domain.LastGeneratedAt,
CreatedAt = domain.CreatedAt,
UpdatedAt = domain.UpdatedAt,
};
}
// ===== Interfaces =====
/// <summary>
/// Store for mirror domain configuration. Initial implementation: in-memory.
/// Future: DB-backed with migration.
/// </summary>
public interface IMirrorDomainStore
{
IReadOnlyList<MirrorDomainRecord> GetAllDomains();
MirrorDomainRecord? GetDomain(string domainId);
Task SaveDomainAsync(MirrorDomainRecord domain, CancellationToken ct = default);
Task DeleteDomainAsync(string domainId, CancellationToken ct = default);
}
/// <summary>
/// Store for mirror global configuration.
/// </summary>
public interface IMirrorConfigStore
{
Task UpdateConfigAsync(UpdateMirrorConfigRequest request, CancellationToken ct = default);
}
// ===== Models =====
public sealed class MirrorDomainRecord
{
public required string Id { get; set; }
public required string DisplayName { get; set; }
public bool RequireAuthentication { get; set; }
public int MaxIndexRequestsPerHour { get; set; } = 120;
public int MaxDownloadRequestsPerHour { get; set; } = 600;
public List<MirrorExportRecord> Exports { get; set; } = [];
public DateTimeOffset CreatedAt { get; set; }
public DateTimeOffset? UpdatedAt { get; set; }
public DateTimeOffset? LastGeneratedAt { get; set; }
public DateTimeOffset? LastGenerateTriggeredAt { get; set; }
public long BundleSizeBytes { get; set; }
public long AdvisoryCount { get; set; }
}
public sealed class MirrorExportRecord
{
public required string Key { get; set; }
public string Format { get; set; } = "json";
public Dictionary<string, string> Filters { get; set; } = new();
}
public sealed class MirrorConfigOptions
{
public string Mode { get; set; } = "direct";
public string? OutputRoot { get; set; }
public string? ConsumerBaseAddress { get; set; }
public bool SigningEnabled { get; set; }
public string? SigningAlgorithm { get; set; }
public string? SigningKeyId { get; set; }
public bool AutoRefreshEnabled { get; set; } = true;
public int RefreshIntervalMinutes { get; set; } = 60;
}
// ===== Request DTOs =====
public sealed record UpdateMirrorConfigRequest
{
public string? Mode { get; init; }
public string? ConsumerBaseAddress { get; init; }
public MirrorSigningRequest? Signing { get; init; }
public bool? AutoRefreshEnabled { get; init; }
public int? RefreshIntervalMinutes { get; init; }
}
public sealed record MirrorSigningRequest
{
public bool Enabled { get; init; }
public string? Algorithm { get; init; }
public string? KeyId { get; init; }
}
public sealed record CreateMirrorDomainRequest
{
public required string Id { get; init; }
public required string DisplayName { get; init; }
public bool RequireAuthentication { get; init; }
public int? MaxIndexRequestsPerHour { get; init; }
public int? MaxDownloadRequestsPerHour { get; init; }
public List<CreateMirrorExportRequest>? Exports { get; init; }
}
public sealed record UpdateMirrorDomainRequest
{
public string? DisplayName { get; init; }
public bool? RequireAuthentication { get; init; }
public int? MaxIndexRequestsPerHour { get; init; }
public int? MaxDownloadRequestsPerHour { get; init; }
public List<CreateMirrorExportRequest>? Exports { get; init; }
}
public sealed record CreateMirrorExportRequest
{
public required string Key { get; init; }
public string? Format { get; init; }
public Dictionary<string, string>? Filters { get; init; }
}
public sealed record MirrorTestRequest
{
public required string BaseAddress { get; init; }
}
// ===== Response DTOs =====
public sealed record MirrorConfigResponse
{
public string Mode { get; init; } = "direct";
public string? OutputRoot { get; init; }
public string? ConsumerBaseAddress { get; init; }
public MirrorSigningResponse? Signing { get; init; }
public bool AutoRefreshEnabled { get; init; }
public int RefreshIntervalMinutes { get; init; }
}
public sealed record MirrorSigningResponse
{
public bool Enabled { get; init; }
public string? Algorithm { get; init; }
public string? KeyId { get; init; }
}
public sealed record MirrorDomainListResponse
{
public IReadOnlyList<MirrorDomainSummary> Domains { get; init; } = [];
public int TotalCount { get; init; }
}
public sealed record MirrorDomainSummary
{
public string Id { get; init; } = "";
public string DisplayName { get; init; } = "";
public int ExportCount { get; init; }
public DateTimeOffset? LastGeneratedAt { get; init; }
public string Staleness { get; init; } = "never_generated";
}
public sealed record MirrorDomainDetailResponse
{
public string Id { get; init; } = "";
public string DisplayName { get; init; } = "";
public bool RequireAuthentication { get; init; }
public int MaxIndexRequestsPerHour { get; init; }
public int MaxDownloadRequestsPerHour { get; init; }
public IReadOnlyList<MirrorExportSummary> Exports { get; init; } = [];
public DateTimeOffset? LastGeneratedAt { get; init; }
public DateTimeOffset CreatedAt { get; init; }
public DateTimeOffset? UpdatedAt { get; init; }
}
public sealed record MirrorExportSummary
{
public string Key { get; init; } = "";
public string Format { get; init; } = "json";
public Dictionary<string, string> Filters { get; init; } = new();
}
public sealed record MirrorDomainStatusResponse
{
public string DomainId { get; init; } = "";
public DateTimeOffset? LastGeneratedAt { get; init; }
public DateTimeOffset? LastGenerateTriggeredAt { get; init; }
public long BundleSizeBytes { get; init; }
public long AdvisoryCount { get; init; }
public int ExportCount { get; init; }
public string Staleness { get; init; } = "never_generated";
}
public sealed record MirrorTestResponse
{
public bool Reachable { get; init; }
public int? StatusCode { get; init; }
public string? Message { get; init; }
}

View File

@@ -0,0 +1,325 @@
using HttpResults = Microsoft.AspNetCore.Http.Results;
using Microsoft.AspNetCore.Mvc;
using StellaOps.Auth.ServerIntegration.Tenancy;
using StellaOps.Concelier.Core.Sources;
namespace StellaOps.Concelier.WebService.Extensions;
/// <summary>
/// Source management endpoints for the advisory source registry.
/// Provides catalog browsing, connectivity checks, and enable/disable controls.
/// </summary>
internal static class SourceManagementEndpointExtensions
{
private const string AdvisoryReadPolicy = "Concelier.Advisories.Read";
private const string SourcesManagePolicy = "Concelier.Sources.Manage";
public static void MapSourceManagementEndpoints(this WebApplication app)
{
var group = app.MapGroup("/api/v1/sources")
.WithTags("Source Management")
.RequireTenant();
// GET /catalog — list all registered source definitions
group.MapGet("/catalog", (
[FromServices] ISourceRegistry registry) =>
{
var sources = registry.GetAllSources();
var items = sources.Select(MapCatalogItem).ToList();
return HttpResults.Ok(new SourceCatalogResponse
{
Items = items,
TotalCount = items.Count
});
})
.WithName("GetSourceCatalog")
.WithSummary("List all registered advisory source definitions")
.WithDescription("Returns the full catalog of advisory data sources with their configuration, endpoints, and default settings.")
.Produces<SourceCatalogResponse>(StatusCodes.Status200OK)
.RequireAuthorization(AdvisoryReadPolicy);
// GET /status — enabled sources with last check results
group.MapGet("/status", async (
[FromServices] ISourceRegistry registry,
CancellationToken cancellationToken) =>
{
var enabledIds = await registry.GetEnabledSourcesAsync(cancellationToken).ConfigureAwait(false);
var allSources = registry.GetAllSources();
var items = new List<SourceStatusItem>(allSources.Count);
foreach (var source in allSources)
{
items.Add(new SourceStatusItem
{
SourceId = source.Id,
Enabled = enabledIds.Contains(source.Id, StringComparer.OrdinalIgnoreCase),
LastCheck = registry.GetLastCheckResult(source.Id)
});
}
return HttpResults.Ok(new SourceStatusResponse { Sources = items });
})
.WithName("GetSourceStatus")
.WithSummary("Get status of all sources with last connectivity check")
.WithDescription("Returns enabled/disabled state and last connectivity check result for every registered source.")
.Produces<SourceStatusResponse>(StatusCodes.Status200OK)
.RequireAuthorization(AdvisoryReadPolicy);
// POST /{sourceId}/enable — enable a single source
group.MapPost("/{sourceId}/enable", async (
string sourceId,
[FromServices] ISourceRegistry registry,
CancellationToken cancellationToken) =>
{
var source = registry.GetSource(sourceId);
if (source is null)
{
return HttpResults.NotFound(new { error = "source_not_found", sourceId });
}
var success = await registry.EnableSourceAsync(sourceId, cancellationToken).ConfigureAwait(false);
return success
? HttpResults.Ok(new { sourceId, enabled = true })
: HttpResults.UnprocessableEntity(new { error = "enable_failed", sourceId });
})
.WithName("EnableSource")
.WithSummary("Enable a source for data ingestion")
.WithDescription("Enables the specified advisory source so it will be included in data ingestion runs.")
.Produces(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.Produces(StatusCodes.Status422UnprocessableEntity)
.RequireAuthorization(SourcesManagePolicy);
// POST /{sourceId}/disable — disable a single source
group.MapPost("/{sourceId}/disable", async (
string sourceId,
[FromServices] ISourceRegistry registry,
CancellationToken cancellationToken) =>
{
var source = registry.GetSource(sourceId);
if (source is null)
{
return HttpResults.NotFound(new { error = "source_not_found", sourceId });
}
var success = await registry.DisableSourceAsync(sourceId, cancellationToken).ConfigureAwait(false);
return success
? HttpResults.Ok(new { sourceId, enabled = false })
: HttpResults.UnprocessableEntity(new { error = "disable_failed", sourceId });
})
.WithName("DisableSource")
.WithSummary("Disable a source")
.WithDescription("Disables the specified advisory source so it will be excluded from data ingestion runs.")
.Produces(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.Produces(StatusCodes.Status422UnprocessableEntity)
.RequireAuthorization(SourcesManagePolicy);
// POST /check — check all sources and auto-configure
group.MapPost("/check", async (
[FromServices] ISourceRegistry registry,
CancellationToken cancellationToken) =>
{
var result = await registry.CheckAllAndAutoConfigureAsync(cancellationToken).ConfigureAwait(false);
return HttpResults.Ok(result);
})
.WithName("CheckAllSources")
.WithSummary("Check connectivity for all sources and auto-configure")
.WithDescription("Runs connectivity checks against all registered sources. Healthy sources are auto-enabled; failed sources are disabled.")
.Produces<SourceCheckResult>(StatusCodes.Status200OK)
.RequireAuthorization(SourcesManagePolicy);
// POST /{sourceId}/check — check connectivity for a single source
group.MapPost("/{sourceId}/check", async (
string sourceId,
[FromServices] ISourceRegistry registry,
CancellationToken cancellationToken) =>
{
var source = registry.GetSource(sourceId);
if (source is null)
{
return HttpResults.NotFound(new { error = "source_not_found", sourceId });
}
var result = await registry.CheckConnectivityAsync(sourceId, cancellationToken).ConfigureAwait(false);
return HttpResults.Ok(result);
})
.WithName("CheckSourceConnectivity")
.WithSummary("Check connectivity for a single source")
.WithDescription("Runs a connectivity check against the specified source and returns detailed status, latency, and remediation steps if failed.")
.Produces<SourceConnectivityResult>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(SourcesManagePolicy);
// POST /batch-enable — enable multiple sources
group.MapPost("/batch-enable", async (
[FromBody] BatchSourceRequest request,
[FromServices] ISourceRegistry registry,
CancellationToken cancellationToken) =>
{
if (request.SourceIds is null || request.SourceIds.Count == 0)
{
return HttpResults.BadRequest(new { error = "source_ids_required" });
}
var results = new List<BatchSourceResultItem>(request.SourceIds.Count);
foreach (var id in request.SourceIds)
{
var source = registry.GetSource(id);
if (source is null)
{
results.Add(new BatchSourceResultItem { SourceId = id, Success = false, Error = "source_not_found" });
continue;
}
var success = await registry.EnableSourceAsync(id, cancellationToken).ConfigureAwait(false);
results.Add(new BatchSourceResultItem { SourceId = id, Success = success, Error = success ? null : "enable_failed" });
}
return HttpResults.Ok(new BatchSourceResponse { Results = results });
})
.WithName("BatchEnableSources")
.WithSummary("Enable multiple sources in a single request")
.WithDescription("Enables each specified source for data ingestion. Returns per-source success/failure results.")
.Produces<BatchSourceResponse>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status400BadRequest)
.RequireAuthorization(SourcesManagePolicy);
// POST /batch-disable — disable multiple sources
group.MapPost("/batch-disable", async (
[FromBody] BatchSourceRequest request,
[FromServices] ISourceRegistry registry,
CancellationToken cancellationToken) =>
{
if (request.SourceIds is null || request.SourceIds.Count == 0)
{
return HttpResults.BadRequest(new { error = "source_ids_required" });
}
var results = new List<BatchSourceResultItem>(request.SourceIds.Count);
foreach (var id in request.SourceIds)
{
var source = registry.GetSource(id);
if (source is null)
{
results.Add(new BatchSourceResultItem { SourceId = id, Success = false, Error = "source_not_found" });
continue;
}
var success = await registry.DisableSourceAsync(id, cancellationToken).ConfigureAwait(false);
results.Add(new BatchSourceResultItem { SourceId = id, Success = success, Error = success ? null : "disable_failed" });
}
return HttpResults.Ok(new BatchSourceResponse { Results = results });
})
.WithName("BatchDisableSources")
.WithSummary("Disable multiple sources in a single request")
.WithDescription("Disables each specified source. Returns per-source success/failure results.")
.Produces<BatchSourceResponse>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status400BadRequest)
.RequireAuthorization(SourcesManagePolicy);
// GET /{sourceId}/check-result — get last check result for a source
group.MapGet("/{sourceId}/check-result", (
string sourceId,
[FromServices] ISourceRegistry registry) =>
{
var source = registry.GetSource(sourceId);
if (source is null)
{
return HttpResults.NotFound(new { error = "source_not_found", sourceId });
}
var result = registry.GetLastCheckResult(sourceId);
if (result is null)
{
return HttpResults.Ok(new { sourceId, lastCheck = (SourceConnectivityResult?)null, message = "no_check_performed" });
}
return HttpResults.Ok(result);
})
.WithName("GetSourceCheckResult")
.WithSummary("Get last connectivity check result for a source")
.WithDescription("Returns the most recent connectivity check result for the specified source, including status, latency, and any error details.")
.Produces<SourceConnectivityResult>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(AdvisoryReadPolicy);
}
private static SourceCatalogItem MapCatalogItem(SourceDefinition source)
{
return new SourceCatalogItem
{
Id = source.Id,
DisplayName = source.DisplayName,
Category = source.Category.ToString(),
Type = source.Type.ToString(),
Description = source.Description,
BaseEndpoint = source.BaseEndpoint,
RequiresAuth = source.RequiresAuthentication,
CredentialEnvVar = source.CredentialEnvVar,
CredentialUrl = source.CredentialUrl,
DocumentationUrl = source.DocumentationUrl,
DefaultPriority = source.DefaultPriority,
Regions = source.Regions,
Tags = source.Tags,
EnabledByDefault = source.EnabledByDefault
};
}
}
// ===== Response DTOs =====
public sealed record SourceCatalogResponse
{
public IReadOnlyList<SourceCatalogItem> Items { get; init; } = [];
public int TotalCount { get; init; }
}
public sealed record SourceCatalogItem
{
public string Id { get; init; } = string.Empty;
public string DisplayName { get; init; } = string.Empty;
public string Category { get; init; } = string.Empty;
public string Type { get; init; } = string.Empty;
public string Description { get; init; } = string.Empty;
public string BaseEndpoint { get; init; } = string.Empty;
public bool RequiresAuth { get; init; }
public string? CredentialEnvVar { get; init; }
public string? CredentialUrl { get; init; }
public string? DocumentationUrl { get; init; }
public int DefaultPriority { get; init; }
public IReadOnlyList<string> Regions { get; init; } = [];
public IReadOnlyList<string> Tags { get; init; } = [];
public bool EnabledByDefault { get; init; }
}
public sealed record SourceStatusResponse
{
public IReadOnlyList<SourceStatusItem> Sources { get; init; } = [];
}
public sealed record SourceStatusItem
{
public string SourceId { get; init; } = string.Empty;
public bool Enabled { get; init; }
public SourceConnectivityResult? LastCheck { get; init; }
}
public sealed record BatchSourceRequest
{
public IReadOnlyList<string> SourceIds { get; init; } = [];
}
public sealed record BatchSourceResponse
{
public IReadOnlyList<BatchSourceResultItem> Results { get; init; } = [];
}
public sealed record BatchSourceResultItem
{
public string SourceId { get; init; } = string.Empty;
public bool Success { get; init; }
public string? Error { get; init; }
}

View File

@@ -35,6 +35,7 @@ using StellaOps.Concelier.Core.Observations;
using StellaOps.Concelier.Core.Orchestration;
using StellaOps.Concelier.Core.Raw;
using StellaOps.Concelier.Core.Signals;
using StellaOps.Concelier.Core.Sources;
using StellaOps.Concelier.Merge;
using StellaOps.Concelier.Merge.Services;
using StellaOps.Concelier.Models;
@@ -55,6 +56,7 @@ using StellaOps.Concelier.WebService.Results;
using StellaOps.Concelier.WebService.Services;
using StellaOps.Concelier.WebService.Telemetry;
using StellaOps.Configuration;
using StellaOps.Excititor.Core;
using StellaOps.Plugin.DependencyInjection;
using StellaOps.Plugin.Hosting;
using StellaOps.Provenance;
@@ -557,6 +559,20 @@ builder.Services.AddConcelierOrchestrationServices();
// Register federation snapshot coordination services (SPRINT_20260208_035)
builder.Services.AddConcelierFederationServices();
// Register advisory source registry and connectivity services
builder.Services.AddSourcesRegistry(builder.Configuration);
// Mirror domain management (in-memory store, future: DB-backed)
builder.Services.AddSingleton<InMemoryMirrorDomainStore>();
builder.Services.AddSingleton<IMirrorDomainStore>(sp => sp.GetRequiredService<InMemoryMirrorDomainStore>());
builder.Services.AddSingleton<IMirrorConfigStore>(sp => sp.GetRequiredService<InMemoryMirrorDomainStore>());
builder.Services.Configure<MirrorConfigOptions>(builder.Configuration.GetSection("Mirror"));
builder.Services.AddHttpClient("MirrorTest");
// Mirror distribution options binding and export scheduler (background bundle refresh, TASK-006b)
builder.Services.Configure<MirrorDistributionOptions>(builder.Configuration.GetSection(MirrorDistributionOptions.SectionName));
builder.Services.AddHostedService<MirrorExportScheduler>();
var features = concelierOptions.Features ?? new ConcelierOptions.FeaturesOptions();
if (!features.NoMergeEnabled)
@@ -931,6 +947,7 @@ app.MapConcelierMirrorEndpoints(authorityConfigured, enforceAuthority);
// Canonical advisory endpoints (Sprint 8200.0012.0003)
app.MapCanonicalAdvisoryEndpoints();
app.MapAdvisorySourceEndpoints();
app.MapSourceManagementEndpoints();
app.MapInterestScoreEndpoints();
// Federation endpoints for site-to-site bundle sync
@@ -945,6 +962,9 @@ app.MapFeedSnapshotEndpoints();
// Feed mirror management, bundles, version locks, offline status
app.MapFeedMirrorManagementEndpoints();
// Mirror domain management CRUD endpoints
app.MapMirrorDomainManagementEndpoints();
app.MapGet("/.well-known/openapi", ([FromServices] OpenApiDiscoveryDocumentProvider provider, HttpContext context) =>
{
var (payload, etag) = provider.GetDocument();

View File

@@ -0,0 +1,34 @@
using System.Collections.Concurrent;
using StellaOps.Concelier.WebService.Extensions;
namespace StellaOps.Concelier.WebService.Services;
/// <summary>
/// In-memory implementation of <see cref="IMirrorDomainStore"/> and <see cref="IMirrorConfigStore"/>.
/// Suitable for development and single-instance deployments. Future: replace with DB-backed store.
/// </summary>
public sealed class InMemoryMirrorDomainStore : IMirrorDomainStore, IMirrorConfigStore
{
private readonly ConcurrentDictionary<string, MirrorDomainRecord> _domains = new(StringComparer.OrdinalIgnoreCase);
public IReadOnlyList<MirrorDomainRecord> GetAllDomains() => _domains.Values.ToList();
public MirrorDomainRecord? GetDomain(string domainId) => _domains.GetValueOrDefault(domainId);
public Task SaveDomainAsync(MirrorDomainRecord domain, CancellationToken ct = default)
{
_domains[domain.Id] = domain;
return Task.CompletedTask;
}
public Task DeleteDomainAsync(string domainId, CancellationToken ct = default)
{
_domains.TryRemove(domainId, out _);
return Task.CompletedTask;
}
public Task UpdateConfigAsync(UpdateMirrorConfigRequest request, CancellationToken ct = default)
{
return Task.CompletedTask;
}
}

View File

@@ -23,6 +23,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../__Libraries/StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Concelier.Interest/StellaOps.Concelier.Interest.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Concelier.SbomIntegration/StellaOps.Concelier.SbomIntegration.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Concelier.Persistence/StellaOps.Concelier.Persistence.csproj" />