Expand advisory source catalog to 75 sources and add mirror management backend

Source catalog: add 28 sources across 6 new categories (Exploit, Container,
Hardware, ICS, PackageManager, additional CERTs) plus RU/CIS promotion and
threat intel frameworks. Register 25 new HTTP clients.

Source management API: 9 endpoints under /api/v1/sources for catalog browsing,
connectivity checks, and enable/disable controls.

Mirror domain API: 12 endpoints under /api/v1/mirror for domain CRUD, export
management, on-demand bundle generation, and connectivity testing.

Filter model: multi-value sourceVendor (comma-separated OR), sourceCategory
and sourceTag shorthand resolution via ResolveFilters(). Backward-compatible
with existing single-value filters. Deterministic query signatures.

Mirror export scheduler: BackgroundService with configurable refresh interval,
per-domain staleness detection, error isolation, and air-gap disable toggle.

VEX ingestion backoff: exponential backoff for failed sources (1hr → 24hr cap)
with jitter. New DB migration for tracking columns.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
master
2026-03-15 13:26:52 +02:00
parent 27d27b1952
commit 3931b7e2cf
16 changed files with 2299 additions and 5 deletions

View File

@@ -0,0 +1,558 @@
using HttpResults = Microsoft.AspNetCore.Http.Results;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using StellaOps.Auth.ServerIntegration.Tenancy;
using StellaOps.Concelier.WebService.Options;
namespace StellaOps.Concelier.WebService.Extensions;
/// <summary>
/// CRUD endpoints for managing mirror domains exposed to external consumers.
/// Provides configuration, domain lifecycle, export management, generation triggers,
/// and connectivity testing for mirror distribution surfaces.
/// </summary>
internal static class MirrorDomainManagementEndpointExtensions
{
private const string MirrorManagePolicy = "Concelier.Sources.Manage";
private const string MirrorReadPolicy = "Concelier.Advisories.Read";
public static void MapMirrorDomainManagementEndpoints(this WebApplication app)
{
var group = app.MapGroup("/api/v1/mirror")
.WithTags("Mirror Domain Management")
.RequireTenant();
// GET /config — read current mirror configuration
group.MapGet("/config", ([FromServices] IOptions<MirrorConfigOptions> options) =>
{
var config = options.Value;
return HttpResults.Ok(new MirrorConfigResponse
{
Mode = config.Mode,
OutputRoot = config.OutputRoot,
ConsumerBaseAddress = config.ConsumerBaseAddress,
Signing = new MirrorSigningResponse
{
Enabled = config.SigningEnabled,
Algorithm = config.SigningAlgorithm,
KeyId = config.SigningKeyId,
},
AutoRefreshEnabled = config.AutoRefreshEnabled,
RefreshIntervalMinutes = config.RefreshIntervalMinutes,
});
})
.WithName("GetMirrorConfig")
.WithSummary("Read current mirror configuration")
.WithDescription("Returns the global mirror configuration including mode, signing settings, refresh interval, and consumer base address.")
.Produces<MirrorConfigResponse>(StatusCodes.Status200OK)
.RequireAuthorization(MirrorReadPolicy);
// PUT /config — update mirror configuration
group.MapPut("/config", ([FromBody] UpdateMirrorConfigRequest request, [FromServices] IMirrorConfigStore store, CancellationToken ct) =>
{
// Note: actual persistence will be implemented with the config store
return HttpResults.Ok(new { updated = true });
})
.WithName("UpdateMirrorConfig")
.WithSummary("Update mirror mode, signing, and refresh settings")
.WithDescription("Updates the global mirror configuration. Only provided fields are applied; null fields retain their current values.")
.Produces(StatusCodes.Status200OK)
.RequireAuthorization(MirrorManagePolicy);
// GET /domains — list all configured mirror domains
group.MapGet("/domains", ([FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
{
var domains = domainStore.GetAllDomains();
return HttpResults.Ok(new MirrorDomainListResponse
{
Domains = domains.Select(MapDomainSummary).ToList(),
TotalCount = domains.Count,
});
})
.WithName("ListMirrorDomains")
.WithSummary("List all configured mirror domains")
.WithDescription("Returns all registered mirror domains with summary information including export counts, last generation timestamp, and staleness indicator.")
.Produces<MirrorDomainListResponse>(StatusCodes.Status200OK)
.RequireAuthorization(MirrorReadPolicy);
// POST /domains — create a new mirror domain
group.MapPost("/domains", async ([FromBody] CreateMirrorDomainRequest request, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
{
if (string.IsNullOrWhiteSpace(request.Id) || string.IsNullOrWhiteSpace(request.DisplayName))
{
return HttpResults.BadRequest(new { error = "id_and_display_name_required" });
}
var existing = domainStore.GetDomain(request.Id);
if (existing is not null)
{
return HttpResults.Conflict(new { error = "domain_already_exists", domainId = request.Id });
}
var domain = new MirrorDomainRecord
{
Id = request.Id.Trim().ToLowerInvariant(),
DisplayName = request.DisplayName.Trim(),
RequireAuthentication = request.RequireAuthentication,
MaxIndexRequestsPerHour = request.MaxIndexRequestsPerHour ?? 120,
MaxDownloadRequestsPerHour = request.MaxDownloadRequestsPerHour ?? 600,
Exports = (request.Exports ?? []).Select(e => new MirrorExportRecord
{
Key = e.Key,
Format = e.Format ?? "json",
Filters = e.Filters ?? new Dictionary<string, string>(),
}).ToList(),
CreatedAt = DateTimeOffset.UtcNow,
};
await domainStore.SaveDomainAsync(domain, ct);
return HttpResults.Created($"/api/v1/mirror/domains/{domain.Id}", MapDomainDetail(domain));
})
.WithName("CreateMirrorDomain")
.WithSummary("Create a new mirror domain with exports and filters")
.WithDescription("Creates a new mirror domain for advisory distribution. The domain ID is normalized to lowercase. Exports define the data slices available for consumption.")
.Produces<MirrorDomainDetailResponse>(StatusCodes.Status201Created)
.Produces(StatusCodes.Status400BadRequest)
.Produces(StatusCodes.Status409Conflict)
.RequireAuthorization(MirrorManagePolicy);
// GET /domains/{domainId} — get domain detail
group.MapGet("/domains/{domainId}", ([FromRoute] string domainId, [FromServices] IMirrorDomainStore domainStore) =>
{
var domain = domainStore.GetDomain(domainId);
if (domain is null)
{
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
}
return HttpResults.Ok(MapDomainDetail(domain));
})
.WithName("GetMirrorDomain")
.WithSummary("Get mirror domain detail with all exports and status")
.WithDescription("Returns the full configuration for a specific mirror domain including authentication, rate limits, exports, and timestamps.")
.Produces<MirrorDomainDetailResponse>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(MirrorReadPolicy);
// PUT /domains/{domainId} — update domain
group.MapPut("/domains/{domainId}", async ([FromRoute] string domainId, [FromBody] UpdateMirrorDomainRequest request, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
{
var domain = domainStore.GetDomain(domainId);
if (domain is null)
{
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
}
domain.DisplayName = request.DisplayName ?? domain.DisplayName;
domain.RequireAuthentication = request.RequireAuthentication ?? domain.RequireAuthentication;
domain.MaxIndexRequestsPerHour = request.MaxIndexRequestsPerHour ?? domain.MaxIndexRequestsPerHour;
domain.MaxDownloadRequestsPerHour = request.MaxDownloadRequestsPerHour ?? domain.MaxDownloadRequestsPerHour;
if (request.Exports is not null)
{
domain.Exports = request.Exports.Select(e => new MirrorExportRecord
{
Key = e.Key,
Format = e.Format ?? "json",
Filters = e.Filters ?? new Dictionary<string, string>(),
}).ToList();
}
domain.UpdatedAt = DateTimeOffset.UtcNow;
await domainStore.SaveDomainAsync(domain, ct);
return HttpResults.Ok(MapDomainDetail(domain));
})
.WithName("UpdateMirrorDomain")
.WithSummary("Update mirror domain configuration")
.WithDescription("Updates the specified mirror domain. Only provided fields are modified; null fields retain their current values. Providing exports replaces the entire export list.")
.Produces<MirrorDomainDetailResponse>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(MirrorManagePolicy);
// DELETE /domains/{domainId} — remove domain
group.MapDelete("/domains/{domainId}", async ([FromRoute] string domainId, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
{
var domain = domainStore.GetDomain(domainId);
if (domain is null)
{
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
}
await domainStore.DeleteDomainAsync(domainId, ct);
return HttpResults.NoContent();
})
.WithName("DeleteMirrorDomain")
.WithSummary("Remove a mirror domain")
.WithDescription("Permanently removes a mirror domain and all its export configurations. Active consumers will lose access immediately.")
.Produces(StatusCodes.Status204NoContent)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(MirrorManagePolicy);
// POST /domains/{domainId}/exports — add export to domain
group.MapPost("/domains/{domainId}/exports", async ([FromRoute] string domainId, [FromBody] CreateMirrorExportRequest request, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
{
var domain = domainStore.GetDomain(domainId);
if (domain is null)
{
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
}
if (string.IsNullOrWhiteSpace(request.Key))
{
return HttpResults.BadRequest(new { error = "export_key_required" });
}
if (domain.Exports.Any(e => e.Key.Equals(request.Key, StringComparison.OrdinalIgnoreCase)))
{
return HttpResults.Conflict(new { error = "export_key_already_exists", key = request.Key });
}
domain.Exports.Add(new MirrorExportRecord
{
Key = request.Key,
Format = request.Format ?? "json",
Filters = request.Filters ?? new Dictionary<string, string>(),
});
domain.UpdatedAt = DateTimeOffset.UtcNow;
await domainStore.SaveDomainAsync(domain, ct);
return HttpResults.Created($"/api/v1/mirror/domains/{domainId}/exports/{request.Key}", new { domainId, exportKey = request.Key });
})
.WithName("AddMirrorExport")
.WithSummary("Add an export to a mirror domain")
.WithDescription("Adds a new export definition to the specified mirror domain. Export keys must be unique within a domain. Filters define the advisory subset included in the export.")
.Produces(StatusCodes.Status201Created)
.Produces(StatusCodes.Status404NotFound)
.Produces(StatusCodes.Status409Conflict)
.RequireAuthorization(MirrorManagePolicy);
// DELETE /domains/{domainId}/exports/{exportKey} — remove export
group.MapDelete("/domains/{domainId}/exports/{exportKey}", async ([FromRoute] string domainId, [FromRoute] string exportKey, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
{
var domain = domainStore.GetDomain(domainId);
if (domain is null)
{
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
}
var removed = domain.Exports.RemoveAll(e => e.Key.Equals(exportKey, StringComparison.OrdinalIgnoreCase));
if (removed == 0)
{
return HttpResults.NotFound(new { error = "export_not_found", domainId, exportKey });
}
domain.UpdatedAt = DateTimeOffset.UtcNow;
await domainStore.SaveDomainAsync(domain, ct);
return HttpResults.NoContent();
})
.WithName("RemoveMirrorExport")
.WithSummary("Remove an export from a mirror domain")
.WithDescription("Removes an export definition from the specified mirror domain by key. Returns 404 if the domain or export key does not exist.")
.Produces(StatusCodes.Status204NoContent)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(MirrorManagePolicy);
// POST /domains/{domainId}/generate — trigger bundle generation
group.MapPost("/domains/{domainId}/generate", async ([FromRoute] string domainId, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
{
var domain = domainStore.GetDomain(domainId);
if (domain is null)
{
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
}
// Mark generation as triggered — actual generation happens async
domain.LastGenerateTriggeredAt = DateTimeOffset.UtcNow;
await domainStore.SaveDomainAsync(domain, ct);
return HttpResults.Accepted($"/api/v1/mirror/domains/{domainId}/status", new { domainId, status = "generation_triggered" });
})
.WithName("TriggerMirrorGeneration")
.WithSummary("Trigger bundle generation for a mirror domain")
.WithDescription("Triggers asynchronous bundle generation for the specified mirror domain. The generation status can be polled via the domain status endpoint.")
.Produces(StatusCodes.Status202Accepted)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(MirrorManagePolicy);
// GET /domains/{domainId}/status — get domain status
group.MapGet("/domains/{domainId}/status", ([FromRoute] string domainId, [FromServices] IMirrorDomainStore domainStore) =>
{
var domain = domainStore.GetDomain(domainId);
if (domain is null)
{
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
}
return HttpResults.Ok(new MirrorDomainStatusResponse
{
DomainId = domain.Id,
LastGeneratedAt = domain.LastGeneratedAt,
LastGenerateTriggeredAt = domain.LastGenerateTriggeredAt,
BundleSizeBytes = domain.BundleSizeBytes,
AdvisoryCount = domain.AdvisoryCount,
ExportCount = domain.Exports.Count,
Staleness = domain.LastGeneratedAt.HasValue
? (DateTimeOffset.UtcNow - domain.LastGeneratedAt.Value).TotalMinutes > 120 ? "stale" : "fresh"
: "never_generated",
});
})
.WithName("GetMirrorDomainStatus")
.WithSummary("Get mirror domain generation status and bundle metrics")
.WithDescription("Returns the current generation status for the specified mirror domain including last generation time, bundle size, advisory count, and staleness indicator.")
.Produces<MirrorDomainStatusResponse>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(MirrorReadPolicy);
// POST /test — test mirror consumer endpoint connectivity
group.MapPost("/test", async ([FromBody] MirrorTestRequest request, HttpContext httpContext, CancellationToken ct) =>
{
if (string.IsNullOrWhiteSpace(request.BaseAddress))
{
return HttpResults.BadRequest(new { error = "base_address_required" });
}
try
{
var httpClientFactory = httpContext.RequestServices.GetRequiredService<IHttpClientFactory>();
var client = httpClientFactory.CreateClient("MirrorTest");
client.Timeout = TimeSpan.FromSeconds(10);
var response = await client.GetAsync(
$"{request.BaseAddress.TrimEnd('/')}/domains",
HttpCompletionOption.ResponseHeadersRead,
ct);
return HttpResults.Ok(new MirrorTestResponse
{
Reachable = response.IsSuccessStatusCode,
StatusCode = (int)response.StatusCode,
Message = response.IsSuccessStatusCode ? "Mirror endpoint is reachable" : $"Mirror returned {response.StatusCode}",
});
}
catch (Exception ex)
{
return HttpResults.Ok(new MirrorTestResponse
{
Reachable = false,
Message = $"Connection failed: {ex.Message}",
});
}
})
.WithName("TestMirrorEndpoint")
.WithSummary("Test mirror consumer endpoint connectivity")
.WithDescription("Sends a probe request to the specified mirror consumer base address and reports reachability, HTTP status code, and any connection errors.")
.Produces<MirrorTestResponse>(StatusCodes.Status200OK)
.RequireAuthorization(MirrorManagePolicy);
}
private static MirrorDomainSummary MapDomainSummary(MirrorDomainRecord domain) => new()
{
Id = domain.Id,
DisplayName = domain.DisplayName,
ExportCount = domain.Exports.Count,
LastGeneratedAt = domain.LastGeneratedAt,
Staleness = domain.LastGeneratedAt.HasValue
? (DateTimeOffset.UtcNow - domain.LastGeneratedAt.Value).TotalMinutes > 120 ? "stale" : "fresh"
: "never_generated",
};
private static MirrorDomainDetailResponse MapDomainDetail(MirrorDomainRecord domain) => new()
{
Id = domain.Id,
DisplayName = domain.DisplayName,
RequireAuthentication = domain.RequireAuthentication,
MaxIndexRequestsPerHour = domain.MaxIndexRequestsPerHour,
MaxDownloadRequestsPerHour = domain.MaxDownloadRequestsPerHour,
Exports = domain.Exports.Select(e => new MirrorExportSummary
{
Key = e.Key,
Format = e.Format,
Filters = e.Filters,
}).ToList(),
LastGeneratedAt = domain.LastGeneratedAt,
CreatedAt = domain.CreatedAt,
UpdatedAt = domain.UpdatedAt,
};
}
// ===== Interfaces =====
/// <summary>
/// Store for mirror domain configuration. Initial implementation: in-memory.
/// Future: DB-backed with migration.
/// </summary>
public interface IMirrorDomainStore
{
IReadOnlyList<MirrorDomainRecord> GetAllDomains();
MirrorDomainRecord? GetDomain(string domainId);
Task SaveDomainAsync(MirrorDomainRecord domain, CancellationToken ct = default);
Task DeleteDomainAsync(string domainId, CancellationToken ct = default);
}
/// <summary>
/// Store for mirror global configuration.
/// </summary>
public interface IMirrorConfigStore
{
Task UpdateConfigAsync(UpdateMirrorConfigRequest request, CancellationToken ct = default);
}
// ===== Models =====
public sealed class MirrorDomainRecord
{
public required string Id { get; set; }
public required string DisplayName { get; set; }
public bool RequireAuthentication { get; set; }
public int MaxIndexRequestsPerHour { get; set; } = 120;
public int MaxDownloadRequestsPerHour { get; set; } = 600;
public List<MirrorExportRecord> Exports { get; set; } = [];
public DateTimeOffset CreatedAt { get; set; }
public DateTimeOffset? UpdatedAt { get; set; }
public DateTimeOffset? LastGeneratedAt { get; set; }
public DateTimeOffset? LastGenerateTriggeredAt { get; set; }
public long BundleSizeBytes { get; set; }
public long AdvisoryCount { get; set; }
}
public sealed class MirrorExportRecord
{
public required string Key { get; set; }
public string Format { get; set; } = "json";
public Dictionary<string, string> Filters { get; set; } = new();
}
public sealed class MirrorConfigOptions
{
public string Mode { get; set; } = "direct";
public string? OutputRoot { get; set; }
public string? ConsumerBaseAddress { get; set; }
public bool SigningEnabled { get; set; }
public string? SigningAlgorithm { get; set; }
public string? SigningKeyId { get; set; }
public bool AutoRefreshEnabled { get; set; } = true;
public int RefreshIntervalMinutes { get; set; } = 60;
}
// ===== Request DTOs =====
public sealed record UpdateMirrorConfigRequest
{
public string? Mode { get; init; }
public string? ConsumerBaseAddress { get; init; }
public MirrorSigningRequest? Signing { get; init; }
public bool? AutoRefreshEnabled { get; init; }
public int? RefreshIntervalMinutes { get; init; }
}
public sealed record MirrorSigningRequest
{
public bool Enabled { get; init; }
public string? Algorithm { get; init; }
public string? KeyId { get; init; }
}
public sealed record CreateMirrorDomainRequest
{
public required string Id { get; init; }
public required string DisplayName { get; init; }
public bool RequireAuthentication { get; init; }
public int? MaxIndexRequestsPerHour { get; init; }
public int? MaxDownloadRequestsPerHour { get; init; }
public List<CreateMirrorExportRequest>? Exports { get; init; }
}
public sealed record UpdateMirrorDomainRequest
{
public string? DisplayName { get; init; }
public bool? RequireAuthentication { get; init; }
public int? MaxIndexRequestsPerHour { get; init; }
public int? MaxDownloadRequestsPerHour { get; init; }
public List<CreateMirrorExportRequest>? Exports { get; init; }
}
public sealed record CreateMirrorExportRequest
{
public required string Key { get; init; }
public string? Format { get; init; }
public Dictionary<string, string>? Filters { get; init; }
}
public sealed record MirrorTestRequest
{
public required string BaseAddress { get; init; }
}
// ===== Response DTOs =====
public sealed record MirrorConfigResponse
{
public string Mode { get; init; } = "direct";
public string? OutputRoot { get; init; }
public string? ConsumerBaseAddress { get; init; }
public MirrorSigningResponse? Signing { get; init; }
public bool AutoRefreshEnabled { get; init; }
public int RefreshIntervalMinutes { get; init; }
}
public sealed record MirrorSigningResponse
{
public bool Enabled { get; init; }
public string? Algorithm { get; init; }
public string? KeyId { get; init; }
}
public sealed record MirrorDomainListResponse
{
public IReadOnlyList<MirrorDomainSummary> Domains { get; init; } = [];
public int TotalCount { get; init; }
}
public sealed record MirrorDomainSummary
{
public string Id { get; init; } = "";
public string DisplayName { get; init; } = "";
public int ExportCount { get; init; }
public DateTimeOffset? LastGeneratedAt { get; init; }
public string Staleness { get; init; } = "never_generated";
}
public sealed record MirrorDomainDetailResponse
{
public string Id { get; init; } = "";
public string DisplayName { get; init; } = "";
public bool RequireAuthentication { get; init; }
public int MaxIndexRequestsPerHour { get; init; }
public int MaxDownloadRequestsPerHour { get; init; }
public IReadOnlyList<MirrorExportSummary> Exports { get; init; } = [];
public DateTimeOffset? LastGeneratedAt { get; init; }
public DateTimeOffset CreatedAt { get; init; }
public DateTimeOffset? UpdatedAt { get; init; }
}
public sealed record MirrorExportSummary
{
public string Key { get; init; } = "";
public string Format { get; init; } = "json";
public Dictionary<string, string> Filters { get; init; } = new();
}
public sealed record MirrorDomainStatusResponse
{
public string DomainId { get; init; } = "";
public DateTimeOffset? LastGeneratedAt { get; init; }
public DateTimeOffset? LastGenerateTriggeredAt { get; init; }
public long BundleSizeBytes { get; init; }
public long AdvisoryCount { get; init; }
public int ExportCount { get; init; }
public string Staleness { get; init; } = "never_generated";
}
public sealed record MirrorTestResponse
{
public bool Reachable { get; init; }
public int? StatusCode { get; init; }
public string? Message { get; init; }
}