Files
git.stella-ops.org/src/Concelier/StellaOps.Concelier.WebService/Extensions/SourceManagementEndpointExtensions.cs
master 3931b7e2cf Expand advisory source catalog to 75 sources and add mirror management backend
Source catalog: add 28 sources across 6 new categories (Exploit, Container,
Hardware, ICS, PackageManager, additional CERTs) plus RU/CIS promotion and
threat intel frameworks. Register 25 new HTTP clients.

Source management API: 9 endpoints under /api/v1/sources for catalog browsing,
connectivity checks, and enable/disable controls.

Mirror domain API: 12 endpoints under /api/v1/mirror for domain CRUD, export
management, on-demand bundle generation, and connectivity testing.

Filter model: multi-value sourceVendor (comma-separated OR), sourceCategory
and sourceTag shorthand resolution via ResolveFilters(). Backward-compatible
with existing single-value filters. Deterministic query signatures.

Mirror export scheduler: BackgroundService with configurable refresh interval,
per-domain staleness detection, error isolation, and air-gap disable toggle.

VEX ingestion backoff: exponential backoff for failed sources (1hr → 24hr cap)
with jitter. New DB migration for tracking columns.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-15 13:26:52 +02:00

326 lines
14 KiB
C#

using HttpResults = Microsoft.AspNetCore.Http.Results;
using Microsoft.AspNetCore.Mvc;
using StellaOps.Auth.ServerIntegration.Tenancy;
using StellaOps.Concelier.Core.Sources;
namespace StellaOps.Concelier.WebService.Extensions;
/// <summary>
/// Source management endpoints for the advisory source registry.
/// Provides catalog browsing, connectivity checks, and enable/disable controls.
/// </summary>
internal static class SourceManagementEndpointExtensions
{
private const string AdvisoryReadPolicy = "Concelier.Advisories.Read";
private const string SourcesManagePolicy = "Concelier.Sources.Manage";
public static void MapSourceManagementEndpoints(this WebApplication app)
{
var group = app.MapGroup("/api/v1/sources")
.WithTags("Source Management")
.RequireTenant();
// GET /catalog — list all registered source definitions
group.MapGet("/catalog", (
[FromServices] ISourceRegistry registry) =>
{
var sources = registry.GetAllSources();
var items = sources.Select(MapCatalogItem).ToList();
return HttpResults.Ok(new SourceCatalogResponse
{
Items = items,
TotalCount = items.Count
});
})
.WithName("GetSourceCatalog")
.WithSummary("List all registered advisory source definitions")
.WithDescription("Returns the full catalog of advisory data sources with their configuration, endpoints, and default settings.")
.Produces<SourceCatalogResponse>(StatusCodes.Status200OK)
.RequireAuthorization(AdvisoryReadPolicy);
// GET /status — enabled sources with last check results
group.MapGet("/status", async (
[FromServices] ISourceRegistry registry,
CancellationToken cancellationToken) =>
{
var enabledIds = await registry.GetEnabledSourcesAsync(cancellationToken).ConfigureAwait(false);
var allSources = registry.GetAllSources();
var items = new List<SourceStatusItem>(allSources.Count);
foreach (var source in allSources)
{
items.Add(new SourceStatusItem
{
SourceId = source.Id,
Enabled = enabledIds.Contains(source.Id, StringComparer.OrdinalIgnoreCase),
LastCheck = registry.GetLastCheckResult(source.Id)
});
}
return HttpResults.Ok(new SourceStatusResponse { Sources = items });
})
.WithName("GetSourceStatus")
.WithSummary("Get status of all sources with last connectivity check")
.WithDescription("Returns enabled/disabled state and last connectivity check result for every registered source.")
.Produces<SourceStatusResponse>(StatusCodes.Status200OK)
.RequireAuthorization(AdvisoryReadPolicy);
// POST /{sourceId}/enable — enable a single source
group.MapPost("/{sourceId}/enable", async (
string sourceId,
[FromServices] ISourceRegistry registry,
CancellationToken cancellationToken) =>
{
var source = registry.GetSource(sourceId);
if (source is null)
{
return HttpResults.NotFound(new { error = "source_not_found", sourceId });
}
var success = await registry.EnableSourceAsync(sourceId, cancellationToken).ConfigureAwait(false);
return success
? HttpResults.Ok(new { sourceId, enabled = true })
: HttpResults.UnprocessableEntity(new { error = "enable_failed", sourceId });
})
.WithName("EnableSource")
.WithSummary("Enable a source for data ingestion")
.WithDescription("Enables the specified advisory source so it will be included in data ingestion runs.")
.Produces(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.Produces(StatusCodes.Status422UnprocessableEntity)
.RequireAuthorization(SourcesManagePolicy);
// POST /{sourceId}/disable — disable a single source
group.MapPost("/{sourceId}/disable", async (
string sourceId,
[FromServices] ISourceRegistry registry,
CancellationToken cancellationToken) =>
{
var source = registry.GetSource(sourceId);
if (source is null)
{
return HttpResults.NotFound(new { error = "source_not_found", sourceId });
}
var success = await registry.DisableSourceAsync(sourceId, cancellationToken).ConfigureAwait(false);
return success
? HttpResults.Ok(new { sourceId, enabled = false })
: HttpResults.UnprocessableEntity(new { error = "disable_failed", sourceId });
})
.WithName("DisableSource")
.WithSummary("Disable a source")
.WithDescription("Disables the specified advisory source so it will be excluded from data ingestion runs.")
.Produces(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.Produces(StatusCodes.Status422UnprocessableEntity)
.RequireAuthorization(SourcesManagePolicy);
// POST /check — check all sources and auto-configure
group.MapPost("/check", async (
[FromServices] ISourceRegistry registry,
CancellationToken cancellationToken) =>
{
var result = await registry.CheckAllAndAutoConfigureAsync(cancellationToken).ConfigureAwait(false);
return HttpResults.Ok(result);
})
.WithName("CheckAllSources")
.WithSummary("Check connectivity for all sources and auto-configure")
.WithDescription("Runs connectivity checks against all registered sources. Healthy sources are auto-enabled; failed sources are disabled.")
.Produces<SourceCheckResult>(StatusCodes.Status200OK)
.RequireAuthorization(SourcesManagePolicy);
// POST /{sourceId}/check — check connectivity for a single source
group.MapPost("/{sourceId}/check", async (
string sourceId,
[FromServices] ISourceRegistry registry,
CancellationToken cancellationToken) =>
{
var source = registry.GetSource(sourceId);
if (source is null)
{
return HttpResults.NotFound(new { error = "source_not_found", sourceId });
}
var result = await registry.CheckConnectivityAsync(sourceId, cancellationToken).ConfigureAwait(false);
return HttpResults.Ok(result);
})
.WithName("CheckSourceConnectivity")
.WithSummary("Check connectivity for a single source")
.WithDescription("Runs a connectivity check against the specified source and returns detailed status, latency, and remediation steps if failed.")
.Produces<SourceConnectivityResult>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(SourcesManagePolicy);
// POST /batch-enable — enable multiple sources
group.MapPost("/batch-enable", async (
[FromBody] BatchSourceRequest request,
[FromServices] ISourceRegistry registry,
CancellationToken cancellationToken) =>
{
if (request.SourceIds is null || request.SourceIds.Count == 0)
{
return HttpResults.BadRequest(new { error = "source_ids_required" });
}
var results = new List<BatchSourceResultItem>(request.SourceIds.Count);
foreach (var id in request.SourceIds)
{
var source = registry.GetSource(id);
if (source is null)
{
results.Add(new BatchSourceResultItem { SourceId = id, Success = false, Error = "source_not_found" });
continue;
}
var success = await registry.EnableSourceAsync(id, cancellationToken).ConfigureAwait(false);
results.Add(new BatchSourceResultItem { SourceId = id, Success = success, Error = success ? null : "enable_failed" });
}
return HttpResults.Ok(new BatchSourceResponse { Results = results });
})
.WithName("BatchEnableSources")
.WithSummary("Enable multiple sources in a single request")
.WithDescription("Enables each specified source for data ingestion. Returns per-source success/failure results.")
.Produces<BatchSourceResponse>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status400BadRequest)
.RequireAuthorization(SourcesManagePolicy);
// POST /batch-disable — disable multiple sources
group.MapPost("/batch-disable", async (
[FromBody] BatchSourceRequest request,
[FromServices] ISourceRegistry registry,
CancellationToken cancellationToken) =>
{
if (request.SourceIds is null || request.SourceIds.Count == 0)
{
return HttpResults.BadRequest(new { error = "source_ids_required" });
}
var results = new List<BatchSourceResultItem>(request.SourceIds.Count);
foreach (var id in request.SourceIds)
{
var source = registry.GetSource(id);
if (source is null)
{
results.Add(new BatchSourceResultItem { SourceId = id, Success = false, Error = "source_not_found" });
continue;
}
var success = await registry.DisableSourceAsync(id, cancellationToken).ConfigureAwait(false);
results.Add(new BatchSourceResultItem { SourceId = id, Success = success, Error = success ? null : "disable_failed" });
}
return HttpResults.Ok(new BatchSourceResponse { Results = results });
})
.WithName("BatchDisableSources")
.WithSummary("Disable multiple sources in a single request")
.WithDescription("Disables each specified source. Returns per-source success/failure results.")
.Produces<BatchSourceResponse>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status400BadRequest)
.RequireAuthorization(SourcesManagePolicy);
// GET /{sourceId}/check-result — get last check result for a source
group.MapGet("/{sourceId}/check-result", (
string sourceId,
[FromServices] ISourceRegistry registry) =>
{
var source = registry.GetSource(sourceId);
if (source is null)
{
return HttpResults.NotFound(new { error = "source_not_found", sourceId });
}
var result = registry.GetLastCheckResult(sourceId);
if (result is null)
{
return HttpResults.Ok(new { sourceId, lastCheck = (SourceConnectivityResult?)null, message = "no_check_performed" });
}
return HttpResults.Ok(result);
})
.WithName("GetSourceCheckResult")
.WithSummary("Get last connectivity check result for a source")
.WithDescription("Returns the most recent connectivity check result for the specified source, including status, latency, and any error details.")
.Produces<SourceConnectivityResult>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(AdvisoryReadPolicy);
}
private static SourceCatalogItem MapCatalogItem(SourceDefinition source)
{
return new SourceCatalogItem
{
Id = source.Id,
DisplayName = source.DisplayName,
Category = source.Category.ToString(),
Type = source.Type.ToString(),
Description = source.Description,
BaseEndpoint = source.BaseEndpoint,
RequiresAuth = source.RequiresAuthentication,
CredentialEnvVar = source.CredentialEnvVar,
CredentialUrl = source.CredentialUrl,
DocumentationUrl = source.DocumentationUrl,
DefaultPriority = source.DefaultPriority,
Regions = source.Regions,
Tags = source.Tags,
EnabledByDefault = source.EnabledByDefault
};
}
}
// ===== Response DTOs =====
public sealed record SourceCatalogResponse
{
public IReadOnlyList<SourceCatalogItem> Items { get; init; } = [];
public int TotalCount { get; init; }
}
public sealed record SourceCatalogItem
{
public string Id { get; init; } = string.Empty;
public string DisplayName { get; init; } = string.Empty;
public string Category { get; init; } = string.Empty;
public string Type { get; init; } = string.Empty;
public string Description { get; init; } = string.Empty;
public string BaseEndpoint { get; init; } = string.Empty;
public bool RequiresAuth { get; init; }
public string? CredentialEnvVar { get; init; }
public string? CredentialUrl { get; init; }
public string? DocumentationUrl { get; init; }
public int DefaultPriority { get; init; }
public IReadOnlyList<string> Regions { get; init; } = [];
public IReadOnlyList<string> Tags { get; init; } = [];
public bool EnabledByDefault { get; init; }
}
public sealed record SourceStatusResponse
{
public IReadOnlyList<SourceStatusItem> Sources { get; init; } = [];
}
public sealed record SourceStatusItem
{
public string SourceId { get; init; } = string.Empty;
public bool Enabled { get; init; }
public SourceConnectivityResult? LastCheck { get; init; }
}
public sealed record BatchSourceRequest
{
public IReadOnlyList<string> SourceIds { get; init; } = [];
}
public sealed record BatchSourceResponse
{
public IReadOnlyList<BatchSourceResultItem> Results { get; init; } = [];
}
public sealed record BatchSourceResultItem
{
public string SourceId { get; init; } = string.Empty;
public bool Success { get; init; }
public string? Error { get; init; }
}