Expand advisory source catalog to 75 sources and add mirror management backend

Source catalog: add 28 sources across 6 new categories (Exploit, Container,
Hardware, ICS, PackageManager, additional CERTs) plus RU/CIS promotion and
threat intel frameworks. Register 25 new HTTP clients.

Source management API: 9 endpoints under /api/v1/sources for catalog browsing,
connectivity checks, and enable/disable controls.

Mirror domain API: 12 endpoints under /api/v1/mirror for domain CRUD, export
management, on-demand bundle generation, and connectivity testing.

Filter model: multi-value sourceVendor (comma-separated OR), sourceCategory
and sourceTag shorthand resolution via ResolveFilters(). Backward-compatible
with existing single-value filters. Deterministic query signatures.

Mirror export scheduler: BackgroundService with configurable refresh interval,
per-domain staleness detection, error isolation, and air-gap disable toggle.

VEX ingestion backoff: exponential backoff for failed sources (1hr → 24hr cap)
with jitter. New DB migration for tracking columns.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
master
2026-03-15 13:26:52 +02:00
parent 27d27b1952
commit 3931b7e2cf
16 changed files with 2299 additions and 5 deletions

View File

@@ -0,0 +1,558 @@
using HttpResults = Microsoft.AspNetCore.Http.Results;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using StellaOps.Auth.ServerIntegration.Tenancy;
using StellaOps.Concelier.WebService.Options;
namespace StellaOps.Concelier.WebService.Extensions;
/// <summary>
/// CRUD endpoints for managing mirror domains exposed to external consumers.
/// Provides configuration, domain lifecycle, export management, generation triggers,
/// and connectivity testing for mirror distribution surfaces.
/// </summary>
internal static class MirrorDomainManagementEndpointExtensions
{
private const string MirrorManagePolicy = "Concelier.Sources.Manage";
private const string MirrorReadPolicy = "Concelier.Advisories.Read";
public static void MapMirrorDomainManagementEndpoints(this WebApplication app)
{
var group = app.MapGroup("/api/v1/mirror")
.WithTags("Mirror Domain Management")
.RequireTenant();
// GET /config — read current mirror configuration
group.MapGet("/config", ([FromServices] IOptions<MirrorConfigOptions> options) =>
{
var config = options.Value;
return HttpResults.Ok(new MirrorConfigResponse
{
Mode = config.Mode,
OutputRoot = config.OutputRoot,
ConsumerBaseAddress = config.ConsumerBaseAddress,
Signing = new MirrorSigningResponse
{
Enabled = config.SigningEnabled,
Algorithm = config.SigningAlgorithm,
KeyId = config.SigningKeyId,
},
AutoRefreshEnabled = config.AutoRefreshEnabled,
RefreshIntervalMinutes = config.RefreshIntervalMinutes,
});
})
.WithName("GetMirrorConfig")
.WithSummary("Read current mirror configuration")
.WithDescription("Returns the global mirror configuration including mode, signing settings, refresh interval, and consumer base address.")
.Produces<MirrorConfigResponse>(StatusCodes.Status200OK)
.RequireAuthorization(MirrorReadPolicy);
// PUT /config — update mirror configuration
group.MapPut("/config", ([FromBody] UpdateMirrorConfigRequest request, [FromServices] IMirrorConfigStore store, CancellationToken ct) =>
{
// Note: actual persistence will be implemented with the config store
return HttpResults.Ok(new { updated = true });
})
.WithName("UpdateMirrorConfig")
.WithSummary("Update mirror mode, signing, and refresh settings")
.WithDescription("Updates the global mirror configuration. Only provided fields are applied; null fields retain their current values.")
.Produces(StatusCodes.Status200OK)
.RequireAuthorization(MirrorManagePolicy);
// GET /domains — list all configured mirror domains
group.MapGet("/domains", ([FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
{
var domains = domainStore.GetAllDomains();
return HttpResults.Ok(new MirrorDomainListResponse
{
Domains = domains.Select(MapDomainSummary).ToList(),
TotalCount = domains.Count,
});
})
.WithName("ListMirrorDomains")
.WithSummary("List all configured mirror domains")
.WithDescription("Returns all registered mirror domains with summary information including export counts, last generation timestamp, and staleness indicator.")
.Produces<MirrorDomainListResponse>(StatusCodes.Status200OK)
.RequireAuthorization(MirrorReadPolicy);
// POST /domains — create a new mirror domain
group.MapPost("/domains", async ([FromBody] CreateMirrorDomainRequest request, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
{
if (string.IsNullOrWhiteSpace(request.Id) || string.IsNullOrWhiteSpace(request.DisplayName))
{
return HttpResults.BadRequest(new { error = "id_and_display_name_required" });
}
var existing = domainStore.GetDomain(request.Id);
if (existing is not null)
{
return HttpResults.Conflict(new { error = "domain_already_exists", domainId = request.Id });
}
var domain = new MirrorDomainRecord
{
Id = request.Id.Trim().ToLowerInvariant(),
DisplayName = request.DisplayName.Trim(),
RequireAuthentication = request.RequireAuthentication,
MaxIndexRequestsPerHour = request.MaxIndexRequestsPerHour ?? 120,
MaxDownloadRequestsPerHour = request.MaxDownloadRequestsPerHour ?? 600,
Exports = (request.Exports ?? []).Select(e => new MirrorExportRecord
{
Key = e.Key,
Format = e.Format ?? "json",
Filters = e.Filters ?? new Dictionary<string, string>(),
}).ToList(),
CreatedAt = DateTimeOffset.UtcNow,
};
await domainStore.SaveDomainAsync(domain, ct);
return HttpResults.Created($"/api/v1/mirror/domains/{domain.Id}", MapDomainDetail(domain));
})
.WithName("CreateMirrorDomain")
.WithSummary("Create a new mirror domain with exports and filters")
.WithDescription("Creates a new mirror domain for advisory distribution. The domain ID is normalized to lowercase. Exports define the data slices available for consumption.")
.Produces<MirrorDomainDetailResponse>(StatusCodes.Status201Created)
.Produces(StatusCodes.Status400BadRequest)
.Produces(StatusCodes.Status409Conflict)
.RequireAuthorization(MirrorManagePolicy);
// GET /domains/{domainId} — get domain detail
group.MapGet("/domains/{domainId}", ([FromRoute] string domainId, [FromServices] IMirrorDomainStore domainStore) =>
{
var domain = domainStore.GetDomain(domainId);
if (domain is null)
{
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
}
return HttpResults.Ok(MapDomainDetail(domain));
})
.WithName("GetMirrorDomain")
.WithSummary("Get mirror domain detail with all exports and status")
.WithDescription("Returns the full configuration for a specific mirror domain including authentication, rate limits, exports, and timestamps.")
.Produces<MirrorDomainDetailResponse>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(MirrorReadPolicy);
// PUT /domains/{domainId} — update domain
group.MapPut("/domains/{domainId}", async ([FromRoute] string domainId, [FromBody] UpdateMirrorDomainRequest request, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
{
var domain = domainStore.GetDomain(domainId);
if (domain is null)
{
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
}
domain.DisplayName = request.DisplayName ?? domain.DisplayName;
domain.RequireAuthentication = request.RequireAuthentication ?? domain.RequireAuthentication;
domain.MaxIndexRequestsPerHour = request.MaxIndexRequestsPerHour ?? domain.MaxIndexRequestsPerHour;
domain.MaxDownloadRequestsPerHour = request.MaxDownloadRequestsPerHour ?? domain.MaxDownloadRequestsPerHour;
if (request.Exports is not null)
{
domain.Exports = request.Exports.Select(e => new MirrorExportRecord
{
Key = e.Key,
Format = e.Format ?? "json",
Filters = e.Filters ?? new Dictionary<string, string>(),
}).ToList();
}
domain.UpdatedAt = DateTimeOffset.UtcNow;
await domainStore.SaveDomainAsync(domain, ct);
return HttpResults.Ok(MapDomainDetail(domain));
})
.WithName("UpdateMirrorDomain")
.WithSummary("Update mirror domain configuration")
.WithDescription("Updates the specified mirror domain. Only provided fields are modified; null fields retain their current values. Providing exports replaces the entire export list.")
.Produces<MirrorDomainDetailResponse>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(MirrorManagePolicy);
// DELETE /domains/{domainId} — remove domain
group.MapDelete("/domains/{domainId}", async ([FromRoute] string domainId, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
{
var domain = domainStore.GetDomain(domainId);
if (domain is null)
{
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
}
await domainStore.DeleteDomainAsync(domainId, ct);
return HttpResults.NoContent();
})
.WithName("DeleteMirrorDomain")
.WithSummary("Remove a mirror domain")
.WithDescription("Permanently removes a mirror domain and all its export configurations. Active consumers will lose access immediately.")
.Produces(StatusCodes.Status204NoContent)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(MirrorManagePolicy);
// POST /domains/{domainId}/exports — add export to domain
group.MapPost("/domains/{domainId}/exports", async ([FromRoute] string domainId, [FromBody] CreateMirrorExportRequest request, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
{
var domain = domainStore.GetDomain(domainId);
if (domain is null)
{
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
}
if (string.IsNullOrWhiteSpace(request.Key))
{
return HttpResults.BadRequest(new { error = "export_key_required" });
}
if (domain.Exports.Any(e => e.Key.Equals(request.Key, StringComparison.OrdinalIgnoreCase)))
{
return HttpResults.Conflict(new { error = "export_key_already_exists", key = request.Key });
}
domain.Exports.Add(new MirrorExportRecord
{
Key = request.Key,
Format = request.Format ?? "json",
Filters = request.Filters ?? new Dictionary<string, string>(),
});
domain.UpdatedAt = DateTimeOffset.UtcNow;
await domainStore.SaveDomainAsync(domain, ct);
return HttpResults.Created($"/api/v1/mirror/domains/{domainId}/exports/{request.Key}", new { domainId, exportKey = request.Key });
})
.WithName("AddMirrorExport")
.WithSummary("Add an export to a mirror domain")
.WithDescription("Adds a new export definition to the specified mirror domain. Export keys must be unique within a domain. Filters define the advisory subset included in the export.")
.Produces(StatusCodes.Status201Created)
.Produces(StatusCodes.Status404NotFound)
.Produces(StatusCodes.Status409Conflict)
.RequireAuthorization(MirrorManagePolicy);
// DELETE /domains/{domainId}/exports/{exportKey} — remove export
group.MapDelete("/domains/{domainId}/exports/{exportKey}", async ([FromRoute] string domainId, [FromRoute] string exportKey, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
{
var domain = domainStore.GetDomain(domainId);
if (domain is null)
{
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
}
var removed = domain.Exports.RemoveAll(e => e.Key.Equals(exportKey, StringComparison.OrdinalIgnoreCase));
if (removed == 0)
{
return HttpResults.NotFound(new { error = "export_not_found", domainId, exportKey });
}
domain.UpdatedAt = DateTimeOffset.UtcNow;
await domainStore.SaveDomainAsync(domain, ct);
return HttpResults.NoContent();
})
.WithName("RemoveMirrorExport")
.WithSummary("Remove an export from a mirror domain")
.WithDescription("Removes an export definition from the specified mirror domain by key. Returns 404 if the domain or export key does not exist.")
.Produces(StatusCodes.Status204NoContent)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(MirrorManagePolicy);
// POST /domains/{domainId}/generate — trigger bundle generation
group.MapPost("/domains/{domainId}/generate", async ([FromRoute] string domainId, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
{
var domain = domainStore.GetDomain(domainId);
if (domain is null)
{
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
}
// Mark generation as triggered — actual generation happens async
domain.LastGenerateTriggeredAt = DateTimeOffset.UtcNow;
await domainStore.SaveDomainAsync(domain, ct);
return HttpResults.Accepted($"/api/v1/mirror/domains/{domainId}/status", new { domainId, status = "generation_triggered" });
})
.WithName("TriggerMirrorGeneration")
.WithSummary("Trigger bundle generation for a mirror domain")
.WithDescription("Triggers asynchronous bundle generation for the specified mirror domain. The generation status can be polled via the domain status endpoint.")
.Produces(StatusCodes.Status202Accepted)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(MirrorManagePolicy);
// GET /domains/{domainId}/status — get domain status
group.MapGet("/domains/{domainId}/status", ([FromRoute] string domainId, [FromServices] IMirrorDomainStore domainStore) =>
{
var domain = domainStore.GetDomain(domainId);
if (domain is null)
{
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
}
return HttpResults.Ok(new MirrorDomainStatusResponse
{
DomainId = domain.Id,
LastGeneratedAt = domain.LastGeneratedAt,
LastGenerateTriggeredAt = domain.LastGenerateTriggeredAt,
BundleSizeBytes = domain.BundleSizeBytes,
AdvisoryCount = domain.AdvisoryCount,
ExportCount = domain.Exports.Count,
Staleness = domain.LastGeneratedAt.HasValue
? (DateTimeOffset.UtcNow - domain.LastGeneratedAt.Value).TotalMinutes > 120 ? "stale" : "fresh"
: "never_generated",
});
})
.WithName("GetMirrorDomainStatus")
.WithSummary("Get mirror domain generation status and bundle metrics")
.WithDescription("Returns the current generation status for the specified mirror domain including last generation time, bundle size, advisory count, and staleness indicator.")
.Produces<MirrorDomainStatusResponse>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(MirrorReadPolicy);
// POST /test — test mirror consumer endpoint connectivity
group.MapPost("/test", async ([FromBody] MirrorTestRequest request, HttpContext httpContext, CancellationToken ct) =>
{
if (string.IsNullOrWhiteSpace(request.BaseAddress))
{
return HttpResults.BadRequest(new { error = "base_address_required" });
}
try
{
var httpClientFactory = httpContext.RequestServices.GetRequiredService<IHttpClientFactory>();
var client = httpClientFactory.CreateClient("MirrorTest");
client.Timeout = TimeSpan.FromSeconds(10);
var response = await client.GetAsync(
$"{request.BaseAddress.TrimEnd('/')}/domains",
HttpCompletionOption.ResponseHeadersRead,
ct);
return HttpResults.Ok(new MirrorTestResponse
{
Reachable = response.IsSuccessStatusCode,
StatusCode = (int)response.StatusCode,
Message = response.IsSuccessStatusCode ? "Mirror endpoint is reachable" : $"Mirror returned {response.StatusCode}",
});
}
catch (Exception ex)
{
return HttpResults.Ok(new MirrorTestResponse
{
Reachable = false,
Message = $"Connection failed: {ex.Message}",
});
}
})
.WithName("TestMirrorEndpoint")
.WithSummary("Test mirror consumer endpoint connectivity")
.WithDescription("Sends a probe request to the specified mirror consumer base address and reports reachability, HTTP status code, and any connection errors.")
.Produces<MirrorTestResponse>(StatusCodes.Status200OK)
.RequireAuthorization(MirrorManagePolicy);
}
private static MirrorDomainSummary MapDomainSummary(MirrorDomainRecord domain) => new()
{
Id = domain.Id,
DisplayName = domain.DisplayName,
ExportCount = domain.Exports.Count,
LastGeneratedAt = domain.LastGeneratedAt,
Staleness = domain.LastGeneratedAt.HasValue
? (DateTimeOffset.UtcNow - domain.LastGeneratedAt.Value).TotalMinutes > 120 ? "stale" : "fresh"
: "never_generated",
};
private static MirrorDomainDetailResponse MapDomainDetail(MirrorDomainRecord domain) => new()
{
Id = domain.Id,
DisplayName = domain.DisplayName,
RequireAuthentication = domain.RequireAuthentication,
MaxIndexRequestsPerHour = domain.MaxIndexRequestsPerHour,
MaxDownloadRequestsPerHour = domain.MaxDownloadRequestsPerHour,
Exports = domain.Exports.Select(e => new MirrorExportSummary
{
Key = e.Key,
Format = e.Format,
Filters = e.Filters,
}).ToList(),
LastGeneratedAt = domain.LastGeneratedAt,
CreatedAt = domain.CreatedAt,
UpdatedAt = domain.UpdatedAt,
};
}
// ===== Interfaces =====
/// <summary>
/// Store for mirror domain configuration. Initial implementation: in-memory.
/// Future: DB-backed with migration.
/// </summary>
public interface IMirrorDomainStore
{
IReadOnlyList<MirrorDomainRecord> GetAllDomains();
MirrorDomainRecord? GetDomain(string domainId);
Task SaveDomainAsync(MirrorDomainRecord domain, CancellationToken ct = default);
Task DeleteDomainAsync(string domainId, CancellationToken ct = default);
}
/// <summary>
/// Store for mirror global configuration.
/// </summary>
public interface IMirrorConfigStore
{
Task UpdateConfigAsync(UpdateMirrorConfigRequest request, CancellationToken ct = default);
}
// ===== Models =====
public sealed class MirrorDomainRecord
{
public required string Id { get; set; }
public required string DisplayName { get; set; }
public bool RequireAuthentication { get; set; }
public int MaxIndexRequestsPerHour { get; set; } = 120;
public int MaxDownloadRequestsPerHour { get; set; } = 600;
public List<MirrorExportRecord> Exports { get; set; } = [];
public DateTimeOffset CreatedAt { get; set; }
public DateTimeOffset? UpdatedAt { get; set; }
public DateTimeOffset? LastGeneratedAt { get; set; }
public DateTimeOffset? LastGenerateTriggeredAt { get; set; }
public long BundleSizeBytes { get; set; }
public long AdvisoryCount { get; set; }
}
public sealed class MirrorExportRecord
{
public required string Key { get; set; }
public string Format { get; set; } = "json";
public Dictionary<string, string> Filters { get; set; } = new();
}
public sealed class MirrorConfigOptions
{
public string Mode { get; set; } = "direct";
public string? OutputRoot { get; set; }
public string? ConsumerBaseAddress { get; set; }
public bool SigningEnabled { get; set; }
public string? SigningAlgorithm { get; set; }
public string? SigningKeyId { get; set; }
public bool AutoRefreshEnabled { get; set; } = true;
public int RefreshIntervalMinutes { get; set; } = 60;
}
// ===== Request DTOs =====
public sealed record UpdateMirrorConfigRequest
{
public string? Mode { get; init; }
public string? ConsumerBaseAddress { get; init; }
public MirrorSigningRequest? Signing { get; init; }
public bool? AutoRefreshEnabled { get; init; }
public int? RefreshIntervalMinutes { get; init; }
}
public sealed record MirrorSigningRequest
{
public bool Enabled { get; init; }
public string? Algorithm { get; init; }
public string? KeyId { get; init; }
}
public sealed record CreateMirrorDomainRequest
{
public required string Id { get; init; }
public required string DisplayName { get; init; }
public bool RequireAuthentication { get; init; }
public int? MaxIndexRequestsPerHour { get; init; }
public int? MaxDownloadRequestsPerHour { get; init; }
public List<CreateMirrorExportRequest>? Exports { get; init; }
}
public sealed record UpdateMirrorDomainRequest
{
public string? DisplayName { get; init; }
public bool? RequireAuthentication { get; init; }
public int? MaxIndexRequestsPerHour { get; init; }
public int? MaxDownloadRequestsPerHour { get; init; }
public List<CreateMirrorExportRequest>? Exports { get; init; }
}
public sealed record CreateMirrorExportRequest
{
public required string Key { get; init; }
public string? Format { get; init; }
public Dictionary<string, string>? Filters { get; init; }
}
public sealed record MirrorTestRequest
{
public required string BaseAddress { get; init; }
}
// ===== Response DTOs =====
public sealed record MirrorConfigResponse
{
public string Mode { get; init; } = "direct";
public string? OutputRoot { get; init; }
public string? ConsumerBaseAddress { get; init; }
public MirrorSigningResponse? Signing { get; init; }
public bool AutoRefreshEnabled { get; init; }
public int RefreshIntervalMinutes { get; init; }
}
public sealed record MirrorSigningResponse
{
public bool Enabled { get; init; }
public string? Algorithm { get; init; }
public string? KeyId { get; init; }
}
public sealed record MirrorDomainListResponse
{
public IReadOnlyList<MirrorDomainSummary> Domains { get; init; } = [];
public int TotalCount { get; init; }
}
public sealed record MirrorDomainSummary
{
public string Id { get; init; } = "";
public string DisplayName { get; init; } = "";
public int ExportCount { get; init; }
public DateTimeOffset? LastGeneratedAt { get; init; }
public string Staleness { get; init; } = "never_generated";
}
public sealed record MirrorDomainDetailResponse
{
public string Id { get; init; } = "";
public string DisplayName { get; init; } = "";
public bool RequireAuthentication { get; init; }
public int MaxIndexRequestsPerHour { get; init; }
public int MaxDownloadRequestsPerHour { get; init; }
public IReadOnlyList<MirrorExportSummary> Exports { get; init; } = [];
public DateTimeOffset? LastGeneratedAt { get; init; }
public DateTimeOffset CreatedAt { get; init; }
public DateTimeOffset? UpdatedAt { get; init; }
}
public sealed record MirrorExportSummary
{
public string Key { get; init; } = "";
public string Format { get; init; } = "json";
public Dictionary<string, string> Filters { get; init; } = new();
}
public sealed record MirrorDomainStatusResponse
{
public string DomainId { get; init; } = "";
public DateTimeOffset? LastGeneratedAt { get; init; }
public DateTimeOffset? LastGenerateTriggeredAt { get; init; }
public long BundleSizeBytes { get; init; }
public long AdvisoryCount { get; init; }
public int ExportCount { get; init; }
public string Staleness { get; init; } = "never_generated";
}
public sealed record MirrorTestResponse
{
public bool Reachable { get; init; }
public int? StatusCode { get; init; }
public string? Message { get; init; }
}

View File

@@ -0,0 +1,325 @@
using HttpResults = Microsoft.AspNetCore.Http.Results;
using Microsoft.AspNetCore.Mvc;
using StellaOps.Auth.ServerIntegration.Tenancy;
using StellaOps.Concelier.Core.Sources;
namespace StellaOps.Concelier.WebService.Extensions;
/// <summary>
/// Source management endpoints for the advisory source registry.
/// Provides catalog browsing, connectivity checks, and enable/disable controls.
/// </summary>
internal static class SourceManagementEndpointExtensions
{
private const string AdvisoryReadPolicy = "Concelier.Advisories.Read";
private const string SourcesManagePolicy = "Concelier.Sources.Manage";
public static void MapSourceManagementEndpoints(this WebApplication app)
{
var group = app.MapGroup("/api/v1/sources")
.WithTags("Source Management")
.RequireTenant();
// GET /catalog — list all registered source definitions
group.MapGet("/catalog", (
[FromServices] ISourceRegistry registry) =>
{
var sources = registry.GetAllSources();
var items = sources.Select(MapCatalogItem).ToList();
return HttpResults.Ok(new SourceCatalogResponse
{
Items = items,
TotalCount = items.Count
});
})
.WithName("GetSourceCatalog")
.WithSummary("List all registered advisory source definitions")
.WithDescription("Returns the full catalog of advisory data sources with their configuration, endpoints, and default settings.")
.Produces<SourceCatalogResponse>(StatusCodes.Status200OK)
.RequireAuthorization(AdvisoryReadPolicy);
// GET /status — enabled sources with last check results
group.MapGet("/status", async (
[FromServices] ISourceRegistry registry,
CancellationToken cancellationToken) =>
{
var enabledIds = await registry.GetEnabledSourcesAsync(cancellationToken).ConfigureAwait(false);
var allSources = registry.GetAllSources();
var items = new List<SourceStatusItem>(allSources.Count);
foreach (var source in allSources)
{
items.Add(new SourceStatusItem
{
SourceId = source.Id,
Enabled = enabledIds.Contains(source.Id, StringComparer.OrdinalIgnoreCase),
LastCheck = registry.GetLastCheckResult(source.Id)
});
}
return HttpResults.Ok(new SourceStatusResponse { Sources = items });
})
.WithName("GetSourceStatus")
.WithSummary("Get status of all sources with last connectivity check")
.WithDescription("Returns enabled/disabled state and last connectivity check result for every registered source.")
.Produces<SourceStatusResponse>(StatusCodes.Status200OK)
.RequireAuthorization(AdvisoryReadPolicy);
// POST /{sourceId}/enable — enable a single source
group.MapPost("/{sourceId}/enable", async (
string sourceId,
[FromServices] ISourceRegistry registry,
CancellationToken cancellationToken) =>
{
var source = registry.GetSource(sourceId);
if (source is null)
{
return HttpResults.NotFound(new { error = "source_not_found", sourceId });
}
var success = await registry.EnableSourceAsync(sourceId, cancellationToken).ConfigureAwait(false);
return success
? HttpResults.Ok(new { sourceId, enabled = true })
: HttpResults.UnprocessableEntity(new { error = "enable_failed", sourceId });
})
.WithName("EnableSource")
.WithSummary("Enable a source for data ingestion")
.WithDescription("Enables the specified advisory source so it will be included in data ingestion runs.")
.Produces(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.Produces(StatusCodes.Status422UnprocessableEntity)
.RequireAuthorization(SourcesManagePolicy);
// POST /{sourceId}/disable — disable a single source
group.MapPost("/{sourceId}/disable", async (
string sourceId,
[FromServices] ISourceRegistry registry,
CancellationToken cancellationToken) =>
{
var source = registry.GetSource(sourceId);
if (source is null)
{
return HttpResults.NotFound(new { error = "source_not_found", sourceId });
}
var success = await registry.DisableSourceAsync(sourceId, cancellationToken).ConfigureAwait(false);
return success
? HttpResults.Ok(new { sourceId, enabled = false })
: HttpResults.UnprocessableEntity(new { error = "disable_failed", sourceId });
})
.WithName("DisableSource")
.WithSummary("Disable a source")
.WithDescription("Disables the specified advisory source so it will be excluded from data ingestion runs.")
.Produces(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.Produces(StatusCodes.Status422UnprocessableEntity)
.RequireAuthorization(SourcesManagePolicy);
// POST /check — check all sources and auto-configure
group.MapPost("/check", async (
[FromServices] ISourceRegistry registry,
CancellationToken cancellationToken) =>
{
var result = await registry.CheckAllAndAutoConfigureAsync(cancellationToken).ConfigureAwait(false);
return HttpResults.Ok(result);
})
.WithName("CheckAllSources")
.WithSummary("Check connectivity for all sources and auto-configure")
.WithDescription("Runs connectivity checks against all registered sources. Healthy sources are auto-enabled; failed sources are disabled.")
.Produces<SourceCheckResult>(StatusCodes.Status200OK)
.RequireAuthorization(SourcesManagePolicy);
// POST /{sourceId}/check — check connectivity for a single source
group.MapPost("/{sourceId}/check", async (
string sourceId,
[FromServices] ISourceRegistry registry,
CancellationToken cancellationToken) =>
{
var source = registry.GetSource(sourceId);
if (source is null)
{
return HttpResults.NotFound(new { error = "source_not_found", sourceId });
}
var result = await registry.CheckConnectivityAsync(sourceId, cancellationToken).ConfigureAwait(false);
return HttpResults.Ok(result);
})
.WithName("CheckSourceConnectivity")
.WithSummary("Check connectivity for a single source")
.WithDescription("Runs a connectivity check against the specified source and returns detailed status, latency, and remediation steps if failed.")
.Produces<SourceConnectivityResult>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(SourcesManagePolicy);
// POST /batch-enable — enable multiple sources
group.MapPost("/batch-enable", async (
[FromBody] BatchSourceRequest request,
[FromServices] ISourceRegistry registry,
CancellationToken cancellationToken) =>
{
if (request.SourceIds is null || request.SourceIds.Count == 0)
{
return HttpResults.BadRequest(new { error = "source_ids_required" });
}
var results = new List<BatchSourceResultItem>(request.SourceIds.Count);
foreach (var id in request.SourceIds)
{
var source = registry.GetSource(id);
if (source is null)
{
results.Add(new BatchSourceResultItem { SourceId = id, Success = false, Error = "source_not_found" });
continue;
}
var success = await registry.EnableSourceAsync(id, cancellationToken).ConfigureAwait(false);
results.Add(new BatchSourceResultItem { SourceId = id, Success = success, Error = success ? null : "enable_failed" });
}
return HttpResults.Ok(new BatchSourceResponse { Results = results });
})
.WithName("BatchEnableSources")
.WithSummary("Enable multiple sources in a single request")
.WithDescription("Enables each specified source for data ingestion. Returns per-source success/failure results.")
.Produces<BatchSourceResponse>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status400BadRequest)
.RequireAuthorization(SourcesManagePolicy);
// POST /batch-disable — disable multiple sources
group.MapPost("/batch-disable", async (
[FromBody] BatchSourceRequest request,
[FromServices] ISourceRegistry registry,
CancellationToken cancellationToken) =>
{
if (request.SourceIds is null || request.SourceIds.Count == 0)
{
return HttpResults.BadRequest(new { error = "source_ids_required" });
}
var results = new List<BatchSourceResultItem>(request.SourceIds.Count);
foreach (var id in request.SourceIds)
{
var source = registry.GetSource(id);
if (source is null)
{
results.Add(new BatchSourceResultItem { SourceId = id, Success = false, Error = "source_not_found" });
continue;
}
var success = await registry.DisableSourceAsync(id, cancellationToken).ConfigureAwait(false);
results.Add(new BatchSourceResultItem { SourceId = id, Success = success, Error = success ? null : "disable_failed" });
}
return HttpResults.Ok(new BatchSourceResponse { Results = results });
})
.WithName("BatchDisableSources")
.WithSummary("Disable multiple sources in a single request")
.WithDescription("Disables each specified source. Returns per-source success/failure results.")
.Produces<BatchSourceResponse>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status400BadRequest)
.RequireAuthorization(SourcesManagePolicy);
// GET /{sourceId}/check-result — get last check result for a source
group.MapGet("/{sourceId}/check-result", (
string sourceId,
[FromServices] ISourceRegistry registry) =>
{
var source = registry.GetSource(sourceId);
if (source is null)
{
return HttpResults.NotFound(new { error = "source_not_found", sourceId });
}
var result = registry.GetLastCheckResult(sourceId);
if (result is null)
{
return HttpResults.Ok(new { sourceId, lastCheck = (SourceConnectivityResult?)null, message = "no_check_performed" });
}
return HttpResults.Ok(result);
})
.WithName("GetSourceCheckResult")
.WithSummary("Get last connectivity check result for a source")
.WithDescription("Returns the most recent connectivity check result for the specified source, including status, latency, and any error details.")
.Produces<SourceConnectivityResult>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(AdvisoryReadPolicy);
}
private static SourceCatalogItem MapCatalogItem(SourceDefinition source)
{
return new SourceCatalogItem
{
Id = source.Id,
DisplayName = source.DisplayName,
Category = source.Category.ToString(),
Type = source.Type.ToString(),
Description = source.Description,
BaseEndpoint = source.BaseEndpoint,
RequiresAuth = source.RequiresAuthentication,
CredentialEnvVar = source.CredentialEnvVar,
CredentialUrl = source.CredentialUrl,
DocumentationUrl = source.DocumentationUrl,
DefaultPriority = source.DefaultPriority,
Regions = source.Regions,
Tags = source.Tags,
EnabledByDefault = source.EnabledByDefault
};
}
}
// ===== Response DTOs =====
public sealed record SourceCatalogResponse
{
public IReadOnlyList<SourceCatalogItem> Items { get; init; } = [];
public int TotalCount { get; init; }
}
public sealed record SourceCatalogItem
{
public string Id { get; init; } = string.Empty;
public string DisplayName { get; init; } = string.Empty;
public string Category { get; init; } = string.Empty;
public string Type { get; init; } = string.Empty;
public string Description { get; init; } = string.Empty;
public string BaseEndpoint { get; init; } = string.Empty;
public bool RequiresAuth { get; init; }
public string? CredentialEnvVar { get; init; }
public string? CredentialUrl { get; init; }
public string? DocumentationUrl { get; init; }
public int DefaultPriority { get; init; }
public IReadOnlyList<string> Regions { get; init; } = [];
public IReadOnlyList<string> Tags { get; init; } = [];
public bool EnabledByDefault { get; init; }
}
public sealed record SourceStatusResponse
{
public IReadOnlyList<SourceStatusItem> Sources { get; init; } = [];
}
public sealed record SourceStatusItem
{
public string SourceId { get; init; } = string.Empty;
public bool Enabled { get; init; }
public SourceConnectivityResult? LastCheck { get; init; }
}
public sealed record BatchSourceRequest
{
public IReadOnlyList<string> SourceIds { get; init; } = [];
}
public sealed record BatchSourceResponse
{
public IReadOnlyList<BatchSourceResultItem> Results { get; init; } = [];
}
public sealed record BatchSourceResultItem
{
public string SourceId { get; init; } = string.Empty;
public bool Success { get; init; }
public string? Error { get; init; }
}

View File

@@ -35,6 +35,7 @@ using StellaOps.Concelier.Core.Observations;
using StellaOps.Concelier.Core.Orchestration;
using StellaOps.Concelier.Core.Raw;
using StellaOps.Concelier.Core.Signals;
using StellaOps.Concelier.Core.Sources;
using StellaOps.Concelier.Merge;
using StellaOps.Concelier.Merge.Services;
using StellaOps.Concelier.Models;
@@ -55,6 +56,7 @@ using StellaOps.Concelier.WebService.Results;
using StellaOps.Concelier.WebService.Services;
using StellaOps.Concelier.WebService.Telemetry;
using StellaOps.Configuration;
using StellaOps.Excititor.Core;
using StellaOps.Plugin.DependencyInjection;
using StellaOps.Plugin.Hosting;
using StellaOps.Provenance;
@@ -557,6 +559,20 @@ builder.Services.AddConcelierOrchestrationServices();
// Register federation snapshot coordination services (SPRINT_20260208_035)
builder.Services.AddConcelierFederationServices();
// Register advisory source registry and connectivity services
builder.Services.AddSourcesRegistry(builder.Configuration);
// Mirror domain management (in-memory store, future: DB-backed)
builder.Services.AddSingleton<InMemoryMirrorDomainStore>();
builder.Services.AddSingleton<IMirrorDomainStore>(sp => sp.GetRequiredService<InMemoryMirrorDomainStore>());
builder.Services.AddSingleton<IMirrorConfigStore>(sp => sp.GetRequiredService<InMemoryMirrorDomainStore>());
builder.Services.Configure<MirrorConfigOptions>(builder.Configuration.GetSection("Mirror"));
builder.Services.AddHttpClient("MirrorTest");
// Mirror distribution options binding and export scheduler (background bundle refresh, TASK-006b)
builder.Services.Configure<MirrorDistributionOptions>(builder.Configuration.GetSection(MirrorDistributionOptions.SectionName));
builder.Services.AddHostedService<MirrorExportScheduler>();
var features = concelierOptions.Features ?? new ConcelierOptions.FeaturesOptions();
if (!features.NoMergeEnabled)
@@ -931,6 +947,7 @@ app.MapConcelierMirrorEndpoints(authorityConfigured, enforceAuthority);
// Canonical advisory endpoints (Sprint 8200.0012.0003)
app.MapCanonicalAdvisoryEndpoints();
app.MapAdvisorySourceEndpoints();
app.MapSourceManagementEndpoints();
app.MapInterestScoreEndpoints();
// Federation endpoints for site-to-site bundle sync
@@ -945,6 +962,9 @@ app.MapFeedSnapshotEndpoints();
// Feed mirror management, bundles, version locks, offline status
app.MapFeedMirrorManagementEndpoints();
// Mirror domain management CRUD endpoints
app.MapMirrorDomainManagementEndpoints();
app.MapGet("/.well-known/openapi", ([FromServices] OpenApiDiscoveryDocumentProvider provider, HttpContext context) =>
{
var (payload, etag) = provider.GetDocument();

View File

@@ -0,0 +1,34 @@
using System.Collections.Concurrent;
using StellaOps.Concelier.WebService.Extensions;
namespace StellaOps.Concelier.WebService.Services;
/// <summary>
/// In-memory implementation of <see cref="IMirrorDomainStore"/> and <see cref="IMirrorConfigStore"/>.
/// Suitable for development and single-instance deployments. Future: replace with DB-backed store.
/// </summary>
public sealed class InMemoryMirrorDomainStore : IMirrorDomainStore, IMirrorConfigStore
{
private readonly ConcurrentDictionary<string, MirrorDomainRecord> _domains = new(StringComparer.OrdinalIgnoreCase);
public IReadOnlyList<MirrorDomainRecord> GetAllDomains() => _domains.Values.ToList();
public MirrorDomainRecord? GetDomain(string domainId) => _domains.GetValueOrDefault(domainId);
public Task SaveDomainAsync(MirrorDomainRecord domain, CancellationToken ct = default)
{
_domains[domain.Id] = domain;
return Task.CompletedTask;
}
public Task DeleteDomainAsync(string domainId, CancellationToken ct = default)
{
_domains.TryRemove(domainId, out _);
return Task.CompletedTask;
}
public Task UpdateConfigAsync(UpdateMirrorConfigRequest request, CancellationToken ct = default)
{
return Task.CompletedTask;
}
}

View File

@@ -23,6 +23,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../__Libraries/StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Concelier.Interest/StellaOps.Concelier.Interest.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Concelier.SbomIntegration/StellaOps.Concelier.SbomIntegration.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Concelier.Persistence/StellaOps.Concelier.Persistence.csproj" />

View File

@@ -128,6 +128,21 @@ public enum SourceCategory
/// <summary>Exploit and threat intelligence sources.</summary>
Threat,
/// <summary>Exploit databases and PoC sources.</summary>
Exploit,
/// <summary>Container image advisory sources.</summary>
Container,
/// <summary>Hardware and firmware PSIRT advisories.</summary>
Hardware,
/// <summary>Industrial control systems and SCADA advisories.</summary>
Ics,
/// <summary>Package manager native advisory databases (cargo-audit, pip-audit, govulncheck, bundler-audit).</summary>
PackageManager,
/// <summary>StellaOps mirrors.</summary>
Mirror,
@@ -150,7 +165,10 @@ public enum SourceType
LocalFile,
/// <summary>Custom/user-defined source.</summary>
Custom
Custom,
/// <summary>STIX/TAXII protocol feed.</summary>
StixTaxii
}
/// <summary>
@@ -894,6 +912,440 @@ public static class SourceDefinitions
Tags = ImmutableArray.Create("cert", "us", "cisa")
};
// ===== Exploit Databases =====
public static readonly SourceDefinition ExploitDb = new()
{
Id = "exploitdb",
DisplayName = "Exploit-DB",
Category = SourceCategory.Exploit,
Type = SourceType.Upstream,
Description = "Offensive Security Exploit Database",
BaseEndpoint = "https://gitlab.com/exploit-database/exploitdb/-/raw/main/",
HealthCheckEndpoint = "https://gitlab.com/exploit-database/exploitdb",
HttpClientName = "ExploitDbClient",
RequiresAuthentication = false,
DocumentationUrl = "https://www.exploit-db.com/",
DefaultPriority = 110,
Tags = ImmutableArray.Create("exploit", "poc", "offensive")
};
public static readonly SourceDefinition PocGithub = new()
{
Id = "poc-github",
DisplayName = "PoC-in-GitHub",
Category = SourceCategory.Exploit,
Type = SourceType.Upstream,
Description = "GitHub repositories containing vulnerability PoCs",
BaseEndpoint = "https://api.github.com/search/repositories",
HealthCheckEndpoint = "https://api.github.com/zen",
HttpClientName = "PocGithubClient",
RequiresAuthentication = true,
CredentialEnvVar = "GITHUB_PAT",
DefaultPriority = 112,
Tags = ImmutableArray.Create("exploit", "poc", "github")
};
public static readonly SourceDefinition Metasploit = new()
{
Id = "metasploit",
DisplayName = "Metasploit Modules",
Category = SourceCategory.Exploit,
Type = SourceType.Upstream,
Description = "Rapid7 Metasploit Framework vulnerability modules",
BaseEndpoint = "https://raw.githubusercontent.com/rapid7/metasploit-framework/master/",
HealthCheckEndpoint = "https://raw.githubusercontent.com/rapid7/metasploit-framework/master/README.md",
HttpClientName = "MetasploitClient",
RequiresAuthentication = false,
DefaultPriority = 114,
Tags = ImmutableArray.Create("exploit", "metasploit", "rapid7")
};
// ===== Cloud Provider Advisories =====
public static readonly SourceDefinition Aws = new()
{
Id = "aws",
DisplayName = "AWS Security Bulletins",
Category = SourceCategory.Vendor,
Type = SourceType.Upstream,
Description = "Amazon Web Services security bulletins",
BaseEndpoint = "https://aws.amazon.com/security/security-bulletins/",
HealthCheckEndpoint = "https://aws.amazon.com/security/security-bulletins/",
HttpClientName = "AwsClient",
RequiresAuthentication = false,
DefaultPriority = 81,
Tags = ImmutableArray.Create("aws", "vendor", "cloud")
};
public static readonly SourceDefinition Azure = new()
{
Id = "azure",
DisplayName = "Azure Security Advisories",
Category = SourceCategory.Vendor,
Type = SourceType.Upstream,
Description = "Microsoft Azure security advisories",
BaseEndpoint = "https://api.msrc.microsoft.com/sug/v2.0/en-US/",
HealthCheckEndpoint = "https://api.msrc.microsoft.com/sug/v2.0/en-US/affectedProduct",
HttpClientName = "AzureClient",
RequiresAuthentication = false,
DefaultPriority = 82,
Tags = ImmutableArray.Create("azure", "vendor", "cloud", "microsoft")
};
public static readonly SourceDefinition Gcp = new()
{
Id = "gcp",
DisplayName = "GCP Security Bulletins",
Category = SourceCategory.Vendor,
Type = SourceType.Upstream,
Description = "Google Cloud Platform security bulletins",
BaseEndpoint = "https://cloud.google.com/support/bulletins/",
HealthCheckEndpoint = "https://cloud.google.com/support/bulletins/",
HttpClientName = "GcpClient",
RequiresAuthentication = false,
DefaultPriority = 83,
Tags = ImmutableArray.Create("gcp", "vendor", "cloud", "google")
};
// ===== Container Sources =====
public static readonly SourceDefinition DockerOfficial = new()
{
Id = "docker-official",
DisplayName = "Docker Official CVEs",
Category = SourceCategory.Container,
Type = SourceType.Upstream,
Description = "Docker Official Images CVE notices",
BaseEndpoint = "https://hub.docker.com/v2/",
HealthCheckEndpoint = "https://hub.docker.com/v2/",
HttpClientName = "DockerOfficialClient",
RequiresAuthentication = false,
DefaultPriority = 120,
Tags = ImmutableArray.Create("docker", "container", "oci")
};
public static readonly SourceDefinition Chainguard = new()
{
Id = "chainguard",
DisplayName = "Chainguard Advisories",
Category = SourceCategory.Container,
Type = SourceType.Upstream,
Description = "Chainguard hardened image advisories",
BaseEndpoint = "https://images.chainguard.dev/",
HealthCheckEndpoint = "https://images.chainguard.dev/",
HttpClientName = "ChainguardClient",
RequiresAuthentication = false,
DefaultPriority = 122,
Tags = ImmutableArray.Create("chainguard", "container", "hardened")
};
// ===== Hardware/Firmware =====
public static readonly SourceDefinition Intel = new()
{
Id = "intel",
DisplayName = "Intel PSIRT",
Category = SourceCategory.Hardware,
Type = SourceType.Upstream,
Description = "Intel Product Security Incident Response Team",
BaseEndpoint = "https://www.intel.com/content/www/us/en/security-center/default.html",
HealthCheckEndpoint = "https://www.intel.com/content/www/us/en/security-center/default.html",
HttpClientName = "IntelClient",
RequiresAuthentication = false,
DefaultPriority = 130,
Tags = ImmutableArray.Create("intel", "hardware", "firmware", "cpu")
};
public static readonly SourceDefinition Amd = new()
{
Id = "amd",
DisplayName = "AMD Security",
Category = SourceCategory.Hardware,
Type = SourceType.Upstream,
Description = "AMD Product Security advisories",
BaseEndpoint = "https://www.amd.com/en/resources/product-security.html",
HealthCheckEndpoint = "https://www.amd.com/en/resources/product-security.html",
HttpClientName = "AmdClient",
RequiresAuthentication = false,
DefaultPriority = 132,
Tags = ImmutableArray.Create("amd", "hardware", "firmware", "cpu")
};
public static readonly SourceDefinition Arm = new()
{
Id = "arm",
DisplayName = "ARM Security Center",
Category = SourceCategory.Hardware,
Type = SourceType.Upstream,
Description = "ARM Security Center advisories",
BaseEndpoint = "https://developer.arm.com/Arm%20Security%20Center/",
HealthCheckEndpoint = "https://developer.arm.com/Arm%20Security%20Center/",
HttpClientName = "ArmClient",
RequiresAuthentication = false,
DefaultPriority = 134,
Tags = ImmutableArray.Create("arm", "hardware", "firmware", "cpu")
};
public static readonly SourceDefinition Siemens = new()
{
Id = "siemens",
DisplayName = "Siemens ProductCERT",
Category = SourceCategory.Ics,
Type = SourceType.Upstream,
Description = "Siemens Product CERT ICS advisories",
BaseEndpoint = "https://cert-portal.siemens.com/productcert/csaf/",
HealthCheckEndpoint = "https://cert-portal.siemens.com/productcert/",
HttpClientName = "SiemensClient",
RequiresAuthentication = false,
DefaultPriority = 136,
Tags = ImmutableArray.Create("siemens", "ics", "scada", "hardware")
};
// ===== Package Manager Native Advisories =====
public static readonly SourceDefinition RustSec = new()
{
Id = "rustsec",
DisplayName = "RustSec Advisory DB",
Category = SourceCategory.PackageManager,
Type = SourceType.Upstream,
Description = "Rust Security Advisory Database (cargo-audit)",
BaseEndpoint = "https://raw.githubusercontent.com/rustsec/advisory-db/main/",
HealthCheckEndpoint = "https://raw.githubusercontent.com/rustsec/advisory-db/main/README.md",
HttpClientName = "RustSecClient",
RequiresAuthentication = false,
DefaultPriority = 63,
Tags = ImmutableArray.Create("rustsec", "package-manager", "rust", "cargo")
};
public static readonly SourceDefinition PyPa = new()
{
Id = "pypa",
DisplayName = "PyPA Advisory DB",
Category = SourceCategory.PackageManager,
Type = SourceType.Upstream,
Description = "Python Packaging Authority Advisory Database (pip-audit)",
BaseEndpoint = "https://raw.githubusercontent.com/pypa/advisory-database/main/",
HealthCheckEndpoint = "https://raw.githubusercontent.com/pypa/advisory-database/main/README.md",
HttpClientName = "PyPaClient",
RequiresAuthentication = false,
DefaultPriority = 53,
Tags = ImmutableArray.Create("pypa", "package-manager", "python", "pip")
};
public static readonly SourceDefinition GoVuln = new()
{
Id = "govuln",
DisplayName = "Go Vuln DB",
Category = SourceCategory.PackageManager,
Type = SourceType.Upstream,
Description = "Go Vulnerability Database (govulncheck)",
BaseEndpoint = "https://vuln.go.dev/",
HealthCheckEndpoint = "https://vuln.go.dev/",
HttpClientName = "GoVulnClient",
RequiresAuthentication = false,
DefaultPriority = 55,
Tags = ImmutableArray.Create("govuln", "package-manager", "go", "golang")
};
public static readonly SourceDefinition BundlerAudit = new()
{
Id = "bundler-audit",
DisplayName = "Ruby Advisory DB",
Category = SourceCategory.PackageManager,
Type = SourceType.Upstream,
Description = "Ruby Advisory Database (bundler-audit)",
BaseEndpoint = "https://raw.githubusercontent.com/rubysec/ruby-advisory-db/main/",
HealthCheckEndpoint = "https://raw.githubusercontent.com/rubysec/ruby-advisory-db/main/README.md",
HttpClientName = "BundlerAuditClient",
RequiresAuthentication = false,
DefaultPriority = 57,
Tags = ImmutableArray.Create("bundler", "package-manager", "ruby", "rubysec")
};
// ===== Additional CERTs =====
public static readonly SourceDefinition CertUa = new()
{
Id = "cert-ua",
DisplayName = "CERT-UA (Ukraine)",
Category = SourceCategory.Cert,
Type = SourceType.Upstream,
Description = "Ukrainian Computer Emergency Response Team",
BaseEndpoint = "https://cert.gov.ua/",
HealthCheckEndpoint = "https://cert.gov.ua/",
HttpClientName = "CertUaClient",
RequiresAuthentication = false,
Regions = ImmutableArray.Create("UA"),
DefaultPriority = 95,
Tags = ImmutableArray.Create("cert", "ukraine")
};
public static readonly SourceDefinition CertPl = new()
{
Id = "cert-pl",
DisplayName = "CERT.PL (Poland)",
Category = SourceCategory.Cert,
Type = SourceType.Upstream,
Description = "Polish CERT",
BaseEndpoint = "https://cert.pl/en/",
HealthCheckEndpoint = "https://cert.pl/en/",
HttpClientName = "CertPlClient",
RequiresAuthentication = false,
Regions = ImmutableArray.Create("PL", "EU"),
DefaultPriority = 96,
Tags = ImmutableArray.Create("cert", "poland", "eu")
};
public static readonly SourceDefinition AusCert = new()
{
Id = "auscert",
DisplayName = "AusCERT (Australia)",
Category = SourceCategory.Cert,
Type = SourceType.Upstream,
Description = "Australian Cyber Security Centre CERT",
BaseEndpoint = "https://auscert.org.au/",
HealthCheckEndpoint = "https://auscert.org.au/",
HttpClientName = "AusCertClient",
RequiresAuthentication = false,
Regions = ImmutableArray.Create("AU", "APAC"),
DefaultPriority = 97,
Tags = ImmutableArray.Create("cert", "australia", "apac")
};
public static readonly SourceDefinition KrCert = new()
{
Id = "krcert",
DisplayName = "KrCERT/CC (South Korea)",
Category = SourceCategory.Cert,
Type = SourceType.Upstream,
Description = "Korean Computer Emergency Response Team",
BaseEndpoint = "https://www.krcert.or.kr/",
HealthCheckEndpoint = "https://www.krcert.or.kr/",
HttpClientName = "KrCertClient",
RequiresAuthentication = false,
Regions = ImmutableArray.Create("KR", "APAC"),
DefaultPriority = 98,
Tags = ImmutableArray.Create("cert", "korea", "apac")
};
public static readonly SourceDefinition CertIn = new()
{
Id = "cert-in",
DisplayName = "CERT-In (India)",
Category = SourceCategory.Cert,
Type = SourceType.Upstream,
Description = "Indian Computer Emergency Response Team",
BaseEndpoint = "https://www.cert-in.org.in/",
HealthCheckEndpoint = "https://www.cert-in.org.in/",
HttpClientName = "CertInClient",
RequiresAuthentication = false,
Regions = ImmutableArray.Create("IN", "APAC"),
DefaultPriority = 99,
Tags = ImmutableArray.Create("cert", "india", "apac")
};
// ===== Russian/CIS Sources =====
public static readonly SourceDefinition FstecBdu = new()
{
Id = "fstec-bdu",
DisplayName = "FSTEC BDU (Russia)",
Category = SourceCategory.Cert,
Type = SourceType.Upstream,
Description = "Federal Service for Technical and Export Control — Bank of Security Threats",
BaseEndpoint = "https://bdu.fstec.ru/",
HealthCheckEndpoint = "https://bdu.fstec.ru/",
HttpClientName = "FstecBduClient",
RequiresAuthentication = false,
Regions = ImmutableArray.Create("RU", "CIS"),
DefaultPriority = 100,
Tags = ImmutableArray.Create("fstec", "bdu", "russia", "cis")
};
public static readonly SourceDefinition Nkcki = new()
{
Id = "nkcki",
DisplayName = "NKCKI (Russia)",
Category = SourceCategory.Cert,
Type = SourceType.Upstream,
Description = "National Coordination Center for Computer Incidents",
BaseEndpoint = "https://safe-surf.ru/",
HealthCheckEndpoint = "https://safe-surf.ru/",
HttpClientName = "NkckiClient",
RequiresAuthentication = false,
Regions = ImmutableArray.Create("RU", "CIS"),
DefaultPriority = 101,
Tags = ImmutableArray.Create("nkcki", "russia", "cis", "cert")
};
public static readonly SourceDefinition KasperskyIcs = new()
{
Id = "kaspersky-ics",
DisplayName = "Kaspersky ICS-CERT",
Category = SourceCategory.Ics,
Type = SourceType.Upstream,
Description = "Kaspersky Industrial Control Systems CERT",
BaseEndpoint = "https://ics-cert.kaspersky.com/",
HealthCheckEndpoint = "https://ics-cert.kaspersky.com/",
HttpClientName = "KasperskyIcsClient",
RequiresAuthentication = false,
Regions = ImmutableArray.Create("RU", "CIS", "GLOBAL"),
DefaultPriority = 102,
Tags = ImmutableArray.Create("kaspersky", "ics", "russia", "cis", "scada")
};
public static readonly SourceDefinition AstraLinux = new()
{
Id = "astra",
DisplayName = "Astra Linux Security",
Category = SourceCategory.Distribution,
Type = SourceType.Upstream,
Description = "Astra Linux FSTEC-certified distribution security updates",
BaseEndpoint = "https://wiki.astralinux.ru/",
HealthCheckEndpoint = "https://wiki.astralinux.ru/",
HttpClientName = "AstraLinuxClient",
RequiresAuthentication = false,
Regions = ImmutableArray.Create("RU", "CIS"),
DefaultPriority = 48,
Tags = ImmutableArray.Create("astra", "distro", "linux", "fstec", "russia")
};
// ===== Threat Intelligence =====
public static readonly SourceDefinition MitreAttack = new()
{
Id = "mitre-attack",
DisplayName = "MITRE ATT&CK",
Category = SourceCategory.Threat,
Type = SourceType.Upstream,
Description = "MITRE ATT&CK adversary tactics and techniques knowledge base",
BaseEndpoint = "https://raw.githubusercontent.com/mitre/cti/master/",
HealthCheckEndpoint = "https://raw.githubusercontent.com/mitre/cti/master/README.md",
HttpClientName = "MitreAttackClient",
RequiresAuthentication = false,
DocumentationUrl = "https://attack.mitre.org/",
DefaultPriority = 140,
Tags = ImmutableArray.Create("mitre", "attack", "threat-intel", "tactics")
};
public static readonly SourceDefinition MitreD3fend = new()
{
Id = "mitre-d3fend",
DisplayName = "MITRE D3FEND",
Category = SourceCategory.Threat,
Type = SourceType.Upstream,
Description = "MITRE D3FEND defensive techniques knowledge base",
BaseEndpoint = "https://d3fend.mitre.org/api/",
HealthCheckEndpoint = "https://d3fend.mitre.org/api/",
HttpClientName = "MitreD3fendClient",
RequiresAuthentication = false,
DocumentationUrl = "https://d3fend.mitre.org/",
DefaultPriority = 142,
Tags = ImmutableArray.Create("mitre", "d3fend", "threat-intel", "defensive")
};
// ===== StellaOps Mirror =====
public static readonly SourceDefinition StellaMirror = new()
@@ -924,14 +1376,32 @@ public static class SourceDefinitions
Nvd, Osv, Ghsa, Cve, Epss, Kev,
// Vendor advisories
RedHat, Microsoft, Amazon, Google, Oracle, Apple, Cisco, Fortinet, Juniper, Palo, Vmware,
// Cloud provider advisories
Aws, Azure, Gcp,
// Linux distributions
Debian, Ubuntu, Alpine, Suse, Rhel, Centos, Fedora, Arch, Gentoo,
Debian, Ubuntu, Alpine, Suse, Rhel, Centos, Fedora, Arch, Gentoo, AstraLinux,
// Ecosystems
Npm, PyPi, Go, RubyGems, Nuget, Maven, Crates, Packagist, Hex,
// Package manager native
RustSec, PyPa, GoVuln, BundlerAudit,
// CSAF/VEX
Csaf, CsafTc, Vex,
// Exploit databases
ExploitDb, PocGithub, Metasploit,
// Container sources
DockerOfficial, Chainguard,
// Hardware/firmware
Intel, Amd, Arm,
// ICS/SCADA
Siemens, KasperskyIcs,
// CERTs
CertFr, CertDe, CertAt, CertBe, CertCh, CertEu, JpCert, UsCert,
// Additional CERTs
CertUa, CertPl, AusCert, KrCert, CertIn,
// Russian/CIS
FstecBdu, Nkcki,
// Threat intelligence
MitreAttack, MitreD3fend,
// Mirrors
StellaMirror);

View File

@@ -156,6 +156,206 @@ public static class SourcesServiceCollectionExtensions
client.Timeout = TimeSpan.FromSeconds(30);
});
// Exploit-DB client
services.AddHttpClient("ExploitDbClient", client =>
{
client.BaseAddress = new Uri("https://gitlab.com/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
// PoC-in-GitHub client
services.AddHttpClient("PocGithubClient", client =>
{
client.BaseAddress = new Uri("https://api.github.com/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.DefaultRequestHeaders.Add("User-Agent", "StellaOps-Concelier");
client.Timeout = TimeSpan.FromSeconds(30);
});
// Metasploit client
services.AddHttpClient("MetasploitClient", client =>
{
client.BaseAddress = new Uri("https://raw.githubusercontent.com/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
// Cloud provider clients
services.AddHttpClient("AwsClient", client =>
{
client.BaseAddress = new Uri("https://aws.amazon.com/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
services.AddHttpClient("AzureClient", client =>
{
client.BaseAddress = new Uri("https://api.msrc.microsoft.com/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
services.AddHttpClient("GcpClient", client =>
{
client.BaseAddress = new Uri("https://cloud.google.com/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
// Container clients
services.AddHttpClient("DockerOfficialClient", client =>
{
client.BaseAddress = new Uri("https://hub.docker.com/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
services.AddHttpClient("ChainguardClient", client =>
{
client.BaseAddress = new Uri("https://images.chainguard.dev/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
// Hardware clients
services.AddHttpClient("IntelClient", client =>
{
client.BaseAddress = new Uri("https://www.intel.com/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
services.AddHttpClient("AmdClient", client =>
{
client.BaseAddress = new Uri("https://www.amd.com/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
services.AddHttpClient("ArmClient", client =>
{
client.BaseAddress = new Uri("https://developer.arm.com/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
services.AddHttpClient("SiemensClient", client =>
{
client.BaseAddress = new Uri("https://cert-portal.siemens.com/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
// Package manager native clients
services.AddHttpClient("RustSecClient", client =>
{
client.BaseAddress = new Uri("https://raw.githubusercontent.com/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
services.AddHttpClient("PyPaClient", client =>
{
client.BaseAddress = new Uri("https://raw.githubusercontent.com/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
services.AddHttpClient("GoVulnClient", client =>
{
client.BaseAddress = new Uri("https://vuln.go.dev/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
services.AddHttpClient("BundlerAuditClient", client =>
{
client.BaseAddress = new Uri("https://raw.githubusercontent.com/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
// Additional CERT clients
services.AddHttpClient("CertUaClient", client =>
{
client.BaseAddress = new Uri("https://cert.gov.ua/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
services.AddHttpClient("CertPlClient", client =>
{
client.BaseAddress = new Uri("https://cert.pl/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
services.AddHttpClient("AusCertClient", client =>
{
client.BaseAddress = new Uri("https://auscert.org.au/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
services.AddHttpClient("KrCertClient", client =>
{
client.BaseAddress = new Uri("https://www.krcert.or.kr/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
services.AddHttpClient("CertInClient", client =>
{
client.BaseAddress = new Uri("https://www.cert-in.org.in/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
// Russian/CIS clients
services.AddHttpClient("FstecBduClient", client =>
{
client.BaseAddress = new Uri("https://bdu.fstec.ru/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
services.AddHttpClient("NkckiClient", client =>
{
client.BaseAddress = new Uri("https://safe-surf.ru/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
services.AddHttpClient("KasperskyIcsClient", client =>
{
client.BaseAddress = new Uri("https://ics-cert.kaspersky.com/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
services.AddHttpClient("AstraLinuxClient", client =>
{
client.BaseAddress = new Uri("https://wiki.astralinux.ru/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
// Threat intelligence clients
services.AddHttpClient("MitreAttackClient", client =>
{
client.BaseAddress = new Uri("https://raw.githubusercontent.com/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
services.AddHttpClient("MitreD3fendClient", client =>
{
client.BaseAddress = new Uri("https://d3fend.mitre.org/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
// StellaOps Mirror client
services.AddHttpClient("StellaMirrorClient", client =>
{

View File

@@ -1,4 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
namespace StellaOps.Excititor.Core;
@@ -29,6 +31,16 @@ public sealed class MirrorDistributionOptions
/// </summary>
public string? TargetRepository { get; set; }
/// <summary>
/// Interval in minutes for periodic mirror bundle refresh. Default: 60.
/// </summary>
public int RefreshIntervalMinutes { get; set; } = 60;
/// <summary>
/// Whether automatic bundle refresh is enabled. Disable for air-gap imports.
/// </summary>
public bool AutoRefreshEnabled { get; set; } = true;
/// <summary>
/// Signing configuration applied to generated bundle payloads.
/// </summary>
@@ -76,8 +88,83 @@ public sealed class MirrorExportOptions
public int? Offset { get; set; } = null;
public string? View { get; set; } = null;
/// <summary>
/// Resolves filter values, expanding category/tag shorthands and comma-separated values
/// into normalized multi-value lists. Source definitions are required for resolving
/// <c>sourceCategory</c> and <c>sourceTag</c> shorthands; pass <c>null</c> when
/// category/tag expansion is not needed.
/// </summary>
/// <param name="sourceDefinitions">
/// Optional catalog of source definitions used to resolve <c>sourceCategory</c> and
/// <c>sourceTag</c> filter values. When <c>null</c>, those filter keys are treated as
/// plain comma-separated values.
/// </param>
/// <returns>
/// A dictionary mapping each filter key to an ordered list of resolved values.
/// Multi-value filters (comma-separated, category-expanded, tag-expanded) produce
/// multiple entries. Values are sorted alphabetically for deterministic signatures.
/// </returns>
public Dictionary<string, IReadOnlyList<string>> ResolveFilters(
IReadOnlyList<MirrorSourceDefinitionDescriptor>? sourceDefinitions = null)
{
var resolved = new Dictionary<string, IReadOnlyList<string>>(StringComparer.OrdinalIgnoreCase);
foreach (var (key, value) in Filters)
{
if (string.IsNullOrWhiteSpace(value)) continue;
if (key.Equals("sourceCategory", StringComparison.OrdinalIgnoreCase) && sourceDefinitions is not null)
{
// Resolve category to source IDs
var matchingIds = sourceDefinitions
.Where(s => s.Category.Equals(value, StringComparison.OrdinalIgnoreCase))
.Select(s => s.Id)
.OrderBy(id => id, StringComparer.OrdinalIgnoreCase)
.ToList();
if (matchingIds.Count > 0)
{
resolved["sourceVendor"] = matchingIds;
}
}
else if (key.Equals("sourceTag", StringComparison.OrdinalIgnoreCase) && sourceDefinitions is not null)
{
// Resolve tag to source IDs
var matchingIds = sourceDefinitions
.Where(s => s.Tags.Contains(value, StringComparer.OrdinalIgnoreCase))
.Select(s => s.Id)
.OrderBy(id => id, StringComparer.OrdinalIgnoreCase)
.ToList();
if (matchingIds.Count > 0)
{
resolved["sourceVendor"] = matchingIds;
}
}
else
{
// Split comma-separated values
var values = value.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.OrderBy(v => v, StringComparer.OrdinalIgnoreCase)
.ToList();
resolved[key] = values;
}
}
return resolved;
}
}
/// <summary>
/// Lightweight descriptor for source definitions used by <see cref="MirrorExportOptions.ResolveFilters"/>.
/// Decouples Excititor.Core from Concelier.Core while allowing category/tag resolution.
/// </summary>
public sealed record MirrorSourceDefinitionDescriptor(
string Id,
string Category,
IReadOnlyList<string> Tags);
public sealed class MirrorSigningOptions
{
/// <summary>

View File

@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
namespace StellaOps.Excititor.Core;
@@ -11,7 +12,32 @@ public sealed record MirrorExportPlan(
public static class MirrorExportPlanner
{
/// <summary>
/// Builds an export plan from the given options, expanding multi-value and
/// comma-separated filter values into individual <see cref="VexQueryFilter"/> entries.
/// This overload does not resolve <c>sourceCategory</c>/<c>sourceTag</c> shorthands.
/// </summary>
public static bool TryBuild(MirrorExportOptions exportOptions, out MirrorExportPlan plan, out string? error)
=> TryBuild(exportOptions, sourceDefinitions: null, out plan, out error);
/// <summary>
/// Builds an export plan from the given options, expanding multi-value filters,
/// comma-separated values, and optionally resolving <c>sourceCategory</c>/<c>sourceTag</c>
/// shorthands when <paramref name="sourceDefinitions"/> is provided.
/// </summary>
/// <param name="exportOptions">The export options containing raw filter configuration.</param>
/// <param name="sourceDefinitions">
/// Optional source catalog for resolving category/tag shorthands. Pass <c>null</c> to
/// skip category/tag resolution (comma-separated expansion still applies).
/// </param>
/// <param name="plan">The resulting export plan when successful.</param>
/// <param name="error">An error code when the build fails.</param>
/// <returns><c>true</c> if the plan was built successfully; otherwise <c>false</c>.</returns>
public static bool TryBuild(
MirrorExportOptions exportOptions,
IReadOnlyList<MirrorSourceDefinitionDescriptor>? sourceDefinitions,
out MirrorExportPlan plan,
out string? error)
{
if (exportOptions is null)
{
@@ -35,7 +61,9 @@ public static class MirrorExportPlanner
return false;
}
var filters = exportOptions.Filters.Select(pair => new VexQueryFilter(pair.Key, pair.Value));
// Resolve filters: expand comma-separated values and category/tag shorthands
var resolvedFilters = exportOptions.ResolveFilters(sourceDefinitions);
var filters = ExpandResolvedFilters(resolvedFilters);
var sorts = exportOptions.Sort.Select(pair => new VexQuerySort(pair.Key, pair.Value));
var query = VexQuery.Create(filters, sorts, exportOptions.Limit, exportOptions.Offset, exportOptions.View);
var signature = VexQuerySignature.FromQuery(query);
@@ -44,4 +72,23 @@ public static class MirrorExportPlanner
error = null;
return true;
}
/// <summary>
/// Expands resolved multi-value filters into individual <see cref="VexQueryFilter"/>
/// entries. For a key with multiple values (e.g., <c>sourceVendor</c> = ["alpine","debian","ubuntu"]),
/// each value produces a separate <see cref="VexQueryFilter"/> with the same key.
/// This ensures OR-semantics: a source matches if its vendor ID equals ANY of the values.
/// Values are emitted in sorted order for deterministic query signatures.
/// </summary>
internal static IEnumerable<VexQueryFilter> ExpandResolvedFilters(
Dictionary<string, IReadOnlyList<string>> resolvedFilters)
{
foreach (var (key, values) in resolvedFilters.OrderBy(kvp => kvp.Key, StringComparer.Ordinal))
{
foreach (var value in values)
{
yield return new VexQueryFilter(key, value);
}
}
}
}

View File

@@ -0,0 +1,443 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Core.Storage;
namespace StellaOps.Excititor.Core;
/// <summary>
/// Background service that periodically checks configured mirror domains for stale
/// export bundles and triggers regeneration when source data has been updated since
/// the last bundle generation. Designed for air-gap awareness: can be fully disabled
/// via <see cref="MirrorDistributionOptions.AutoRefreshEnabled"/>.
/// </summary>
public sealed class MirrorExportScheduler : BackgroundService
{
private readonly IServiceScopeFactory _scopeFactory;
private readonly IOptionsMonitor<MirrorDistributionOptions> _optionsMonitor;
private readonly ILogger<MirrorExportScheduler> _logger;
private readonly TimeProvider _timeProvider;
private readonly ConcurrentDictionary<string, DomainGenerationStatus> _domainStatus = new(StringComparer.OrdinalIgnoreCase);
/// <summary>
/// Initializes a new instance of <see cref="MirrorExportScheduler"/>.
/// </summary>
public MirrorExportScheduler(
IServiceScopeFactory scopeFactory,
IOptionsMonitor<MirrorDistributionOptions> optionsMonitor,
ILogger<MirrorExportScheduler> logger,
TimeProvider timeProvider)
{
_scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory));
_optionsMonitor = optionsMonitor ?? throw new ArgumentNullException(nameof(optionsMonitor));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
}
/// <summary>
/// Returns a snapshot of generation status for all tracked domains.
/// </summary>
public IReadOnlyDictionary<string, DomainGenerationStatus> GetDomainStatuses()
=> _domainStatus.ToDictionary(kvp => kvp.Key, kvp => kvp.Value, StringComparer.OrdinalIgnoreCase);
/// <inheritdoc />
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var options = _optionsMonitor.CurrentValue;
if (!options.AutoRefreshEnabled)
{
_logger.LogInformation("MirrorExportScheduler is disabled (AutoRefreshEnabled=false). Exiting.");
return;
}
if (!options.Enabled)
{
_logger.LogInformation("MirrorExportScheduler is disabled (Mirror distribution disabled). Exiting.");
return;
}
var intervalMinutes = Math.Max(options.RefreshIntervalMinutes, 1);
_logger.LogInformation(
"MirrorExportScheduler started. Refresh interval: {IntervalMinutes} min, Domains: {DomainCount}",
intervalMinutes,
options.Domains.Count);
// Allow other services to initialize before the first sweep
await Task.Delay(TimeSpan.FromSeconds(15), stoppingToken).ConfigureAwait(false);
while (!stoppingToken.IsCancellationRequested)
{
try
{
var currentOptions = _optionsMonitor.CurrentValue;
if (!currentOptions.AutoRefreshEnabled)
{
_logger.LogInformation("MirrorExportScheduler auto-refresh disabled at runtime. Stopping refresh loop.");
break;
}
if (!currentOptions.Enabled)
{
_logger.LogDebug("Mirror distribution disabled; skipping refresh cycle.");
}
else
{
await RefreshAllDomainsAsync(currentOptions, stoppingToken).ConfigureAwait(false);
}
intervalMinutes = Math.Max(currentOptions.RefreshIntervalMinutes, 1);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "Unhandled error in MirrorExportScheduler refresh cycle.");
}
try
{
await Task.Delay(TimeSpan.FromMinutes(intervalMinutes), stoppingToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
}
_logger.LogInformation("MirrorExportScheduler stopped.");
}
private async Task RefreshAllDomainsAsync(MirrorDistributionOptions options, CancellationToken cancellationToken)
{
if (options.Domains.Count == 0)
{
_logger.LogDebug("No mirror domains configured; skipping refresh cycle.");
return;
}
_logger.LogDebug("Starting mirror refresh cycle for {DomainCount} domain(s).", options.Domains.Count);
foreach (var domain in options.Domains)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
try
{
await RefreshDomainAsync(options, domain, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception ex)
{
_logger.LogError(
ex,
"Failed to refresh mirror domain {DomainId}. Continuing with remaining domains.",
domain.Id);
UpdateDomainStatus(domain.Id, status => status with
{
LastError = ex.Message,
LastErrorAt = _timeProvider.GetUtcNow(),
ConsecutiveFailures = status.ConsecutiveFailures + 1,
});
}
}
_logger.LogDebug("Mirror refresh cycle completed.");
}
private async Task RefreshDomainAsync(
MirrorDistributionOptions options,
MirrorDomainOptions domain,
CancellationToken cancellationToken)
{
if (domain.Exports.Count == 0)
{
_logger.LogDebug("Domain {DomainId} has no exports; skipping.", domain.Id);
return;
}
var stalePlanCount = 0;
var regeneratedCount = 0;
var domainStartTime = Stopwatch.GetTimestamp();
using var scope = _scopeFactory.CreateScope();
var connectorStateRepo = scope.ServiceProvider.GetService<IVexConnectorStateRepository>();
// Resolve the latest source update time from connector state
DateTimeOffset? latestSourceUpdate = null;
if (connectorStateRepo is not null)
{
var connectorStates = await connectorStateRepo.ListAsync(cancellationToken).ConfigureAwait(false);
latestSourceUpdate = connectorStates
.Where(s => s.LastUpdated.HasValue)
.Max(s => s.LastUpdated);
}
foreach (var exportOption in domain.Exports)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
if (!MirrorExportPlanner.TryBuild(exportOption, out var plan, out var error))
{
_logger.LogWarning(
"Skipping export {ExportKey} in domain {DomainId}: {Error}",
exportOption.Key,
domain.Id,
error);
continue;
}
try
{
var isStale = await CheckExportStalenessAsync(
scope.ServiceProvider,
plan,
latestSourceUpdate,
cancellationToken).ConfigureAwait(false);
if (!isStale)
{
_logger.LogDebug(
"Export {ExportKey} in domain {DomainId} is fresh; skipping regeneration.",
plan.Key,
domain.Id);
continue;
}
stalePlanCount++;
_logger.LogInformation(
"Export {ExportKey} in domain {DomainId} is stale. Triggering bundle regeneration.",
plan.Key,
domain.Id);
var exportStartTime = Stopwatch.GetTimestamp();
await RegenerateExportAsync(
scope.ServiceProvider,
options,
domain,
plan,
cancellationToken).ConfigureAwait(false);
var exportDuration = Stopwatch.GetElapsedTime(exportStartTime);
regeneratedCount++;
_logger.LogInformation(
"Regenerated export {ExportKey} in domain {DomainId} in {DurationMs:F0}ms.",
plan.Key,
domain.Id,
exportDuration.TotalMilliseconds);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception ex)
{
_logger.LogError(
ex,
"Failed to regenerate export {ExportKey} in domain {DomainId}. Continuing with remaining exports.",
plan.Key,
domain.Id);
}
}
var domainDuration = Stopwatch.GetElapsedTime(domainStartTime);
var now = _timeProvider.GetUtcNow();
if (regeneratedCount > 0)
{
UpdateDomainStatus(domain.Id, status => status with
{
LastGeneratedAt = now,
LastRefreshDurationMs = domainDuration.TotalMilliseconds,
ExportsRegenerated = regeneratedCount,
StaleExportsFound = stalePlanCount,
ConsecutiveFailures = 0,
LastError = null,
LastErrorAt = null,
});
_logger.LogInformation(
"Domain {DomainId} refresh complete: {Regenerated}/{Stale} exports regenerated in {DurationMs:F0}ms.",
domain.Id,
regeneratedCount,
stalePlanCount,
domainDuration.TotalMilliseconds);
}
else
{
UpdateDomainStatus(domain.Id, status => status with
{
LastCheckedAt = now,
StaleExportsFound = stalePlanCount,
});
}
}
private async Task<bool> CheckExportStalenessAsync(
IServiceProvider services,
MirrorExportPlan plan,
DateTimeOffset? latestSourceUpdate,
CancellationToken cancellationToken)
{
// Resolve IVexExportStore from the DI container at runtime to avoid
// a compile-time dependency from Excititor.Core on Excititor.Export
var exportStore = services.GetService<IMirrorExportManifestLookup>();
if (exportStore is null)
{
// If no export store is registered, treat as stale (first run or unconfigured)
_logger.LogDebug("No IMirrorExportManifestLookup registered; treating export {ExportKey} as stale.", plan.Key);
return true;
}
var lastManifest = await exportStore.FindManifestAsync(plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false);
if (lastManifest is null)
{
// No previous export: definitely stale
return true;
}
// If we have source update information, compare against last export time
if (latestSourceUpdate.HasValue && lastManifest.CreatedAt < latestSourceUpdate.Value)
{
_logger.LogDebug(
"Export {ExportKey} created at {ExportCreatedAt:O} is older than latest source update at {SourceUpdate:O}.",
plan.Key,
lastManifest.CreatedAt,
latestSourceUpdate.Value);
return true;
}
return false;
}
private static async Task RegenerateExportAsync(
IServiceProvider services,
MirrorDistributionOptions options,
MirrorDomainOptions domain,
MirrorExportPlan plan,
CancellationToken cancellationToken)
{
var regenerator = services.GetService<IMirrorBundleRegenerator>();
if (regenerator is null)
{
// No regenerator registered; bundle generation deferred to the export pipeline
return;
}
await regenerator.RegenerateAsync(domain.Id, plan.Key, plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false);
}
private void UpdateDomainStatus(string domainId, Func<DomainGenerationStatus, DomainGenerationStatus> updater)
{
_domainStatus.AddOrUpdate(
domainId,
_ => updater(DomainGenerationStatus.Empty),
(_, existing) => updater(existing));
}
}
/// <summary>
/// Status record tracking the last generation state for a mirror domain.
/// Exposed via <see cref="MirrorExportScheduler.GetDomainStatuses"/> for
/// observability and staleness reporting.
/// </summary>
public sealed record DomainGenerationStatus
{
public static readonly DomainGenerationStatus Empty = new();
/// <summary>When the domain was last successfully regenerated.</summary>
public DateTimeOffset? LastGeneratedAt { get; init; }
/// <summary>When the domain was last checked (even if nothing was stale).</summary>
public DateTimeOffset? LastCheckedAt { get; init; }
/// <summary>Duration of the last refresh cycle in milliseconds.</summary>
public double LastRefreshDurationMs { get; init; }
/// <summary>Number of stale exports found in the last cycle.</summary>
public int StaleExportsFound { get; init; }
/// <summary>Number of exports regenerated in the last cycle.</summary>
public int ExportsRegenerated { get; init; }
/// <summary>Number of consecutive domain-level failures.</summary>
public int ConsecutiveFailures { get; init; }
/// <summary>Last error message, if any.</summary>
public string? LastError { get; init; }
/// <summary>When the last error occurred.</summary>
public DateTimeOffset? LastErrorAt { get; init; }
}
/// <summary>
/// Abstraction for looking up the latest export manifest by signature and format.
/// This decouples <see cref="MirrorExportScheduler"/> (Excititor.Core) from the
/// concrete <c>IVexExportStore</c> in Excititor.Export without introducing a
/// compile-time dependency.
/// </summary>
public interface IMirrorExportManifestLookup
{
/// <summary>
/// Finds the latest export manifest matching the given query signature and format.
/// </summary>
ValueTask<MirrorExportManifestSnapshot?> FindManifestAsync(
VexQuerySignature signature,
VexExportFormat format,
CancellationToken cancellationToken);
}
/// <summary>
/// Lightweight snapshot of an export manifest used for staleness comparisons.
/// </summary>
public sealed record MirrorExportManifestSnapshot(
string ExportId,
DateTimeOffset CreatedAt,
long SizeBytes,
int ClaimCount);
/// <summary>
/// Abstraction for triggering bundle regeneration from the scheduler.
/// Implemented by the export pipeline (Excititor.Export or WebService layer)
/// to perform the actual VEX query, serialization, and bundle publishing.
/// </summary>
public interface IMirrorBundleRegenerator
{
/// <summary>
/// Regenerates the export bundle for the given domain, export key, query signature, and format.
/// </summary>
Task RegenerateAsync(
string domainId,
string exportKey,
VexQuerySignature signature,
VexExportFormat format,
CancellationToken cancellationToken);
}

View File

@@ -15,6 +15,7 @@
<PackageReference Include="Microsoft.Extensions.Http" />
<PackageReference Include="Microsoft.Extensions.Options" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
</ItemGroup>
<ItemGroup>

View File

@@ -49,6 +49,15 @@ public interface IVexSourceRepository
string? errorMessage = null,
CancellationToken cancellationToken = default);
/// <summary>
/// Updates failure tracking fields for exponential backoff.
/// </summary>
Task UpdateFailureTrackingAsync(
string sourceId,
int consecutiveFailures,
DateTimeOffset? nextEligiblePollAt,
CancellationToken cancellationToken = default);
/// <summary>
/// Deletes a source by its ID.
/// </summary>

View File

@@ -68,9 +68,28 @@ public sealed class VexIngestionScheduler : BackgroundService
return;
}
_logger.LogInformation("Found {Count} sources due for polling", dueSources.Count);
var utcNow = DateTimeOffset.UtcNow;
var eligibleSources = dueSources
.Where(s => s.NextEligiblePollAt == null || s.NextEligiblePollAt <= utcNow)
.ToList();
var tasks = dueSources.Select(source => PollSourceWithThrottlingAsync(source, cancellationToken));
var skippedCount = dueSources.Count - eligibleSources.Count;
if (skippedCount > 0)
{
_logger.LogInformation(
"Skipped {SkippedCount} sources due to backoff. {EligibleCount} sources eligible for polling",
skippedCount, eligibleSources.Count);
}
if (eligibleSources.Count == 0)
{
_logger.LogDebug("No eligible sources after backoff filtering");
return;
}
_logger.LogInformation("Found {Count} sources due for polling", eligibleSources.Count);
var tasks = eligibleSources.Select(source => PollSourceWithThrottlingAsync(source, cancellationToken));
await Task.WhenAll(tasks);
}
@@ -100,6 +119,13 @@ public sealed class VexIngestionScheduler : BackgroundService
DateTimeOffset.UtcNow,
null,
cancellationToken);
// Reset backoff on success
if (source.ConsecutiveFailures > 0)
{
await _sourceRepository.UpdateFailureTrackingAsync(
source.SourceId, 0, null, cancellationToken);
}
}
else
{
@@ -113,6 +139,23 @@ public sealed class VexIngestionScheduler : BackgroundService
DateTimeOffset.UtcNow,
result.ErrorMessage,
cancellationToken);
var failures = source.ConsecutiveFailures + 1;
var backoff = _options.SourceBackoff;
var rawBackoff = backoff.InitialBackoffSeconds * Math.Pow(backoff.BackoffMultiplier, failures - 1);
var capped = Math.Min(rawBackoff, backoff.MaxBackoffSeconds);
var jittered = capped + capped * backoff.JitterFactor * (Random.Shared.NextDouble() * 2 - 1);
var nextEligible = DateTimeOffset.UtcNow.AddSeconds(jittered);
if (failures >= backoff.MaxConsecutiveFailures)
{
_logger.LogWarning(
"Source {SourceId} has failed {Failures} consecutive times. Next retry at {NextEligible}",
source.SourceId, failures, nextEligible);
}
await _sourceRepository.UpdateFailureTrackingAsync(
source.SourceId, failures, nextEligible, cancellationToken);
}
}
catch (Exception ex)
@@ -124,6 +167,23 @@ public sealed class VexIngestionScheduler : BackgroundService
DateTimeOffset.UtcNow,
ex.Message,
cancellationToken);
var failures = source.ConsecutiveFailures + 1;
var backoff = _options.SourceBackoff;
var rawBackoff = backoff.InitialBackoffSeconds * Math.Pow(backoff.BackoffMultiplier, failures - 1);
var capped = Math.Min(rawBackoff, backoff.MaxBackoffSeconds);
var jittered = capped + capped * backoff.JitterFactor * (Random.Shared.NextDouble() * 2 - 1);
var nextEligible = DateTimeOffset.UtcNow.AddSeconds(jittered);
if (failures >= backoff.MaxConsecutiveFailures)
{
_logger.LogWarning(
"Source {SourceId} has failed {Failures} consecutive times. Next retry at {NextEligible}",
source.SourceId, failures, nextEligible);
}
await _sourceRepository.UpdateFailureTrackingAsync(
source.SourceId, failures, nextEligible, cancellationToken);
}
finally
{

View File

@@ -68,6 +68,12 @@ public sealed record VexSource
/// When the source configuration was last updated.
/// </summary>
public DateTimeOffset? UpdatedAt { get; init; }
/// <summary>Number of consecutive polling failures.</summary>
public int ConsecutiveFailures { get; init; }
/// <summary>Next eligible poll time after backoff. Null means eligible immediately.</summary>
public DateTimeOffset? NextEligiblePollAt { get; init; }
}
/// <summary>

View File

@@ -71,6 +71,32 @@ public sealed class VexHubOptions
/// Configuration for distribution/export behavior.
/// </summary>
public DistributionOptions Distribution { get; set; } = new();
/// <summary>
/// Configuration for source failure backoff behavior.
/// </summary>
public SourceBackoffOptions SourceBackoff { get; set; } = new();
/// <summary>
/// Configuration for source failure backoff behavior.
/// </summary>
public sealed class SourceBackoffOptions
{
/// <summary>Initial backoff in seconds after first failure.</summary>
public int InitialBackoffSeconds { get; set; } = 3600;
/// <summary>Maximum backoff in seconds.</summary>
public int MaxBackoffSeconds { get; set; } = 86400;
/// <summary>Multiplier for each consecutive failure.</summary>
public double BackoffMultiplier { get; set; } = 2.0;
/// <summary>Jitter factor to avoid thundering herd.</summary>
public double JitterFactor { get; set; } = 0.1;
/// <summary>Max failures before logging a warning (source stays backed off, not disabled).</summary>
public int MaxConsecutiveFailures { get; set; } = 10;
}
}
/// <summary>

View File

@@ -0,0 +1,7 @@
-- Migration: 002_add_source_backoff_columns
-- Sprint: Advisory & VEX Source Management
-- Adds failure tracking columns for exponential backoff
ALTER TABLE vexhub.vex_sources
ADD COLUMN IF NOT EXISTS consecutive_failures INT NOT NULL DEFAULT 0,
ADD COLUMN IF NOT EXISTS next_eligible_poll_at TIMESTAMPTZ NULL;