Source catalog: add 28 sources across 6 new categories (Exploit, Container, Hardware, ICS, PackageManager, additional CERTs) plus RU/CIS promotion and threat intel frameworks. Register 25 new HTTP clients. Source management API: 9 endpoints under /api/v1/sources for catalog browsing, connectivity checks, and enable/disable controls. Mirror domain API: 12 endpoints under /api/v1/mirror for domain CRUD, export management, on-demand bundle generation, and connectivity testing. Filter model: multi-value sourceVendor (comma-separated OR), sourceCategory and sourceTag shorthand resolution via ResolveFilters(). Backward-compatible with existing single-value filters. Deterministic query signatures. Mirror export scheduler: BackgroundService with configurable refresh interval, per-domain staleness detection, error isolation, and air-gap disable toggle. VEX ingestion backoff: exponential backoff for failed sources (1hr → 24hr cap) with jitter. New DB migration for tracking columns. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
326 lines
14 KiB
C#
326 lines
14 KiB
C#
using HttpResults = Microsoft.AspNetCore.Http.Results;
|
|
using Microsoft.AspNetCore.Mvc;
|
|
using StellaOps.Auth.ServerIntegration.Tenancy;
|
|
using StellaOps.Concelier.Core.Sources;
|
|
|
|
namespace StellaOps.Concelier.WebService.Extensions;
|
|
|
|
/// <summary>
|
|
/// Source management endpoints for the advisory source registry.
|
|
/// Provides catalog browsing, connectivity checks, and enable/disable controls.
|
|
/// </summary>
|
|
internal static class SourceManagementEndpointExtensions
|
|
{
|
|
private const string AdvisoryReadPolicy = "Concelier.Advisories.Read";
|
|
private const string SourcesManagePolicy = "Concelier.Sources.Manage";
|
|
|
|
public static void MapSourceManagementEndpoints(this WebApplication app)
|
|
{
|
|
var group = app.MapGroup("/api/v1/sources")
|
|
.WithTags("Source Management")
|
|
.RequireTenant();
|
|
|
|
// GET /catalog — list all registered source definitions
|
|
group.MapGet("/catalog", (
|
|
[FromServices] ISourceRegistry registry) =>
|
|
{
|
|
var sources = registry.GetAllSources();
|
|
var items = sources.Select(MapCatalogItem).ToList();
|
|
|
|
return HttpResults.Ok(new SourceCatalogResponse
|
|
{
|
|
Items = items,
|
|
TotalCount = items.Count
|
|
});
|
|
})
|
|
.WithName("GetSourceCatalog")
|
|
.WithSummary("List all registered advisory source definitions")
|
|
.WithDescription("Returns the full catalog of advisory data sources with their configuration, endpoints, and default settings.")
|
|
.Produces<SourceCatalogResponse>(StatusCodes.Status200OK)
|
|
.RequireAuthorization(AdvisoryReadPolicy);
|
|
|
|
// GET /status — enabled sources with last check results
|
|
group.MapGet("/status", async (
|
|
[FromServices] ISourceRegistry registry,
|
|
CancellationToken cancellationToken) =>
|
|
{
|
|
var enabledIds = await registry.GetEnabledSourcesAsync(cancellationToken).ConfigureAwait(false);
|
|
var allSources = registry.GetAllSources();
|
|
var items = new List<SourceStatusItem>(allSources.Count);
|
|
|
|
foreach (var source in allSources)
|
|
{
|
|
items.Add(new SourceStatusItem
|
|
{
|
|
SourceId = source.Id,
|
|
Enabled = enabledIds.Contains(source.Id, StringComparer.OrdinalIgnoreCase),
|
|
LastCheck = registry.GetLastCheckResult(source.Id)
|
|
});
|
|
}
|
|
|
|
return HttpResults.Ok(new SourceStatusResponse { Sources = items });
|
|
})
|
|
.WithName("GetSourceStatus")
|
|
.WithSummary("Get status of all sources with last connectivity check")
|
|
.WithDescription("Returns enabled/disabled state and last connectivity check result for every registered source.")
|
|
.Produces<SourceStatusResponse>(StatusCodes.Status200OK)
|
|
.RequireAuthorization(AdvisoryReadPolicy);
|
|
|
|
// POST /{sourceId}/enable — enable a single source
|
|
group.MapPost("/{sourceId}/enable", async (
|
|
string sourceId,
|
|
[FromServices] ISourceRegistry registry,
|
|
CancellationToken cancellationToken) =>
|
|
{
|
|
var source = registry.GetSource(sourceId);
|
|
if (source is null)
|
|
{
|
|
return HttpResults.NotFound(new { error = "source_not_found", sourceId });
|
|
}
|
|
|
|
var success = await registry.EnableSourceAsync(sourceId, cancellationToken).ConfigureAwait(false);
|
|
return success
|
|
? HttpResults.Ok(new { sourceId, enabled = true })
|
|
: HttpResults.UnprocessableEntity(new { error = "enable_failed", sourceId });
|
|
})
|
|
.WithName("EnableSource")
|
|
.WithSummary("Enable a source for data ingestion")
|
|
.WithDescription("Enables the specified advisory source so it will be included in data ingestion runs.")
|
|
.Produces(StatusCodes.Status200OK)
|
|
.Produces(StatusCodes.Status404NotFound)
|
|
.Produces(StatusCodes.Status422UnprocessableEntity)
|
|
.RequireAuthorization(SourcesManagePolicy);
|
|
|
|
// POST /{sourceId}/disable — disable a single source
|
|
group.MapPost("/{sourceId}/disable", async (
|
|
string sourceId,
|
|
[FromServices] ISourceRegistry registry,
|
|
CancellationToken cancellationToken) =>
|
|
{
|
|
var source = registry.GetSource(sourceId);
|
|
if (source is null)
|
|
{
|
|
return HttpResults.NotFound(new { error = "source_not_found", sourceId });
|
|
}
|
|
|
|
var success = await registry.DisableSourceAsync(sourceId, cancellationToken).ConfigureAwait(false);
|
|
return success
|
|
? HttpResults.Ok(new { sourceId, enabled = false })
|
|
: HttpResults.UnprocessableEntity(new { error = "disable_failed", sourceId });
|
|
})
|
|
.WithName("DisableSource")
|
|
.WithSummary("Disable a source")
|
|
.WithDescription("Disables the specified advisory source so it will be excluded from data ingestion runs.")
|
|
.Produces(StatusCodes.Status200OK)
|
|
.Produces(StatusCodes.Status404NotFound)
|
|
.Produces(StatusCodes.Status422UnprocessableEntity)
|
|
.RequireAuthorization(SourcesManagePolicy);
|
|
|
|
// POST /check — check all sources and auto-configure
|
|
group.MapPost("/check", async (
|
|
[FromServices] ISourceRegistry registry,
|
|
CancellationToken cancellationToken) =>
|
|
{
|
|
var result = await registry.CheckAllAndAutoConfigureAsync(cancellationToken).ConfigureAwait(false);
|
|
return HttpResults.Ok(result);
|
|
})
|
|
.WithName("CheckAllSources")
|
|
.WithSummary("Check connectivity for all sources and auto-configure")
|
|
.WithDescription("Runs connectivity checks against all registered sources. Healthy sources are auto-enabled; failed sources are disabled.")
|
|
.Produces<SourceCheckResult>(StatusCodes.Status200OK)
|
|
.RequireAuthorization(SourcesManagePolicy);
|
|
|
|
// POST /{sourceId}/check — check connectivity for a single source
|
|
group.MapPost("/{sourceId}/check", async (
|
|
string sourceId,
|
|
[FromServices] ISourceRegistry registry,
|
|
CancellationToken cancellationToken) =>
|
|
{
|
|
var source = registry.GetSource(sourceId);
|
|
if (source is null)
|
|
{
|
|
return HttpResults.NotFound(new { error = "source_not_found", sourceId });
|
|
}
|
|
|
|
var result = await registry.CheckConnectivityAsync(sourceId, cancellationToken).ConfigureAwait(false);
|
|
return HttpResults.Ok(result);
|
|
})
|
|
.WithName("CheckSourceConnectivity")
|
|
.WithSummary("Check connectivity for a single source")
|
|
.WithDescription("Runs a connectivity check against the specified source and returns detailed status, latency, and remediation steps if failed.")
|
|
.Produces<SourceConnectivityResult>(StatusCodes.Status200OK)
|
|
.Produces(StatusCodes.Status404NotFound)
|
|
.RequireAuthorization(SourcesManagePolicy);
|
|
|
|
// POST /batch-enable — enable multiple sources
|
|
group.MapPost("/batch-enable", async (
|
|
[FromBody] BatchSourceRequest request,
|
|
[FromServices] ISourceRegistry registry,
|
|
CancellationToken cancellationToken) =>
|
|
{
|
|
if (request.SourceIds is null || request.SourceIds.Count == 0)
|
|
{
|
|
return HttpResults.BadRequest(new { error = "source_ids_required" });
|
|
}
|
|
|
|
var results = new List<BatchSourceResultItem>(request.SourceIds.Count);
|
|
foreach (var id in request.SourceIds)
|
|
{
|
|
var source = registry.GetSource(id);
|
|
if (source is null)
|
|
{
|
|
results.Add(new BatchSourceResultItem { SourceId = id, Success = false, Error = "source_not_found" });
|
|
continue;
|
|
}
|
|
|
|
var success = await registry.EnableSourceAsync(id, cancellationToken).ConfigureAwait(false);
|
|
results.Add(new BatchSourceResultItem { SourceId = id, Success = success, Error = success ? null : "enable_failed" });
|
|
}
|
|
|
|
return HttpResults.Ok(new BatchSourceResponse { Results = results });
|
|
})
|
|
.WithName("BatchEnableSources")
|
|
.WithSummary("Enable multiple sources in a single request")
|
|
.WithDescription("Enables each specified source for data ingestion. Returns per-source success/failure results.")
|
|
.Produces<BatchSourceResponse>(StatusCodes.Status200OK)
|
|
.Produces(StatusCodes.Status400BadRequest)
|
|
.RequireAuthorization(SourcesManagePolicy);
|
|
|
|
// POST /batch-disable — disable multiple sources
|
|
group.MapPost("/batch-disable", async (
|
|
[FromBody] BatchSourceRequest request,
|
|
[FromServices] ISourceRegistry registry,
|
|
CancellationToken cancellationToken) =>
|
|
{
|
|
if (request.SourceIds is null || request.SourceIds.Count == 0)
|
|
{
|
|
return HttpResults.BadRequest(new { error = "source_ids_required" });
|
|
}
|
|
|
|
var results = new List<BatchSourceResultItem>(request.SourceIds.Count);
|
|
foreach (var id in request.SourceIds)
|
|
{
|
|
var source = registry.GetSource(id);
|
|
if (source is null)
|
|
{
|
|
results.Add(new BatchSourceResultItem { SourceId = id, Success = false, Error = "source_not_found" });
|
|
continue;
|
|
}
|
|
|
|
var success = await registry.DisableSourceAsync(id, cancellationToken).ConfigureAwait(false);
|
|
results.Add(new BatchSourceResultItem { SourceId = id, Success = success, Error = success ? null : "disable_failed" });
|
|
}
|
|
|
|
return HttpResults.Ok(new BatchSourceResponse { Results = results });
|
|
})
|
|
.WithName("BatchDisableSources")
|
|
.WithSummary("Disable multiple sources in a single request")
|
|
.WithDescription("Disables each specified source. Returns per-source success/failure results.")
|
|
.Produces<BatchSourceResponse>(StatusCodes.Status200OK)
|
|
.Produces(StatusCodes.Status400BadRequest)
|
|
.RequireAuthorization(SourcesManagePolicy);
|
|
|
|
// GET /{sourceId}/check-result — get last check result for a source
|
|
group.MapGet("/{sourceId}/check-result", (
|
|
string sourceId,
|
|
[FromServices] ISourceRegistry registry) =>
|
|
{
|
|
var source = registry.GetSource(sourceId);
|
|
if (source is null)
|
|
{
|
|
return HttpResults.NotFound(new { error = "source_not_found", sourceId });
|
|
}
|
|
|
|
var result = registry.GetLastCheckResult(sourceId);
|
|
if (result is null)
|
|
{
|
|
return HttpResults.Ok(new { sourceId, lastCheck = (SourceConnectivityResult?)null, message = "no_check_performed" });
|
|
}
|
|
|
|
return HttpResults.Ok(result);
|
|
})
|
|
.WithName("GetSourceCheckResult")
|
|
.WithSummary("Get last connectivity check result for a source")
|
|
.WithDescription("Returns the most recent connectivity check result for the specified source, including status, latency, and any error details.")
|
|
.Produces<SourceConnectivityResult>(StatusCodes.Status200OK)
|
|
.Produces(StatusCodes.Status404NotFound)
|
|
.RequireAuthorization(AdvisoryReadPolicy);
|
|
}
|
|
|
|
private static SourceCatalogItem MapCatalogItem(SourceDefinition source)
|
|
{
|
|
return new SourceCatalogItem
|
|
{
|
|
Id = source.Id,
|
|
DisplayName = source.DisplayName,
|
|
Category = source.Category.ToString(),
|
|
Type = source.Type.ToString(),
|
|
Description = source.Description,
|
|
BaseEndpoint = source.BaseEndpoint,
|
|
RequiresAuth = source.RequiresAuthentication,
|
|
CredentialEnvVar = source.CredentialEnvVar,
|
|
CredentialUrl = source.CredentialUrl,
|
|
DocumentationUrl = source.DocumentationUrl,
|
|
DefaultPriority = source.DefaultPriority,
|
|
Regions = source.Regions,
|
|
Tags = source.Tags,
|
|
EnabledByDefault = source.EnabledByDefault
|
|
};
|
|
}
|
|
}
|
|
|
|
// ===== Response DTOs =====
|
|
|
|
public sealed record SourceCatalogResponse
|
|
{
|
|
public IReadOnlyList<SourceCatalogItem> Items { get; init; } = [];
|
|
public int TotalCount { get; init; }
|
|
}
|
|
|
|
public sealed record SourceCatalogItem
|
|
{
|
|
public string Id { get; init; } = string.Empty;
|
|
public string DisplayName { get; init; } = string.Empty;
|
|
public string Category { get; init; } = string.Empty;
|
|
public string Type { get; init; } = string.Empty;
|
|
public string Description { get; init; } = string.Empty;
|
|
public string BaseEndpoint { get; init; } = string.Empty;
|
|
public bool RequiresAuth { get; init; }
|
|
public string? CredentialEnvVar { get; init; }
|
|
public string? CredentialUrl { get; init; }
|
|
public string? DocumentationUrl { get; init; }
|
|
public int DefaultPriority { get; init; }
|
|
public IReadOnlyList<string> Regions { get; init; } = [];
|
|
public IReadOnlyList<string> Tags { get; init; } = [];
|
|
public bool EnabledByDefault { get; init; }
|
|
}
|
|
|
|
public sealed record SourceStatusResponse
|
|
{
|
|
public IReadOnlyList<SourceStatusItem> Sources { get; init; } = [];
|
|
}
|
|
|
|
public sealed record SourceStatusItem
|
|
{
|
|
public string SourceId { get; init; } = string.Empty;
|
|
public bool Enabled { get; init; }
|
|
public SourceConnectivityResult? LastCheck { get; init; }
|
|
}
|
|
|
|
public sealed record BatchSourceRequest
|
|
{
|
|
public IReadOnlyList<string> SourceIds { get; init; } = [];
|
|
}
|
|
|
|
public sealed record BatchSourceResponse
|
|
{
|
|
public IReadOnlyList<BatchSourceResultItem> Results { get; init; } = [];
|
|
}
|
|
|
|
public sealed record BatchSourceResultItem
|
|
{
|
|
public string SourceId { get; init; } = string.Empty;
|
|
public bool Success { get; init; }
|
|
public string? Error { get; init; }
|
|
}
|