Expand advisory source catalog to 75 sources and add mirror management backend
Source catalog: add 28 sources across 6 new categories (Exploit, Container, Hardware, ICS, PackageManager, additional CERTs) plus RU/CIS promotion and threat intel frameworks. Register 25 new HTTP clients. Source management API: 9 endpoints under /api/v1/sources for catalog browsing, connectivity checks, and enable/disable controls. Mirror domain API: 12 endpoints under /api/v1/mirror for domain CRUD, export management, on-demand bundle generation, and connectivity testing. Filter model: multi-value sourceVendor (comma-separated OR), sourceCategory and sourceTag shorthand resolution via ResolveFilters(). Backward-compatible with existing single-value filters. Deterministic query signatures. Mirror export scheduler: BackgroundService with configurable refresh interval, per-domain staleness detection, error isolation, and air-gap disable toggle. VEX ingestion backoff: exponential backoff for failed sources (1hr → 24hr cap) with jitter. New DB migration for tracking columns. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,558 @@
|
||||
using HttpResults = Microsoft.AspNetCore.Http.Results;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Auth.ServerIntegration.Tenancy;
|
||||
using StellaOps.Concelier.WebService.Options;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Extensions;
|
||||
|
||||
/// <summary>
|
||||
/// CRUD endpoints for managing mirror domains exposed to external consumers.
|
||||
/// Provides configuration, domain lifecycle, export management, generation triggers,
|
||||
/// and connectivity testing for mirror distribution surfaces.
|
||||
/// </summary>
|
||||
internal static class MirrorDomainManagementEndpointExtensions
|
||||
{
|
||||
private const string MirrorManagePolicy = "Concelier.Sources.Manage";
|
||||
private const string MirrorReadPolicy = "Concelier.Advisories.Read";
|
||||
|
||||
public static void MapMirrorDomainManagementEndpoints(this WebApplication app)
|
||||
{
|
||||
var group = app.MapGroup("/api/v1/mirror")
|
||||
.WithTags("Mirror Domain Management")
|
||||
.RequireTenant();
|
||||
|
||||
// GET /config — read current mirror configuration
|
||||
group.MapGet("/config", ([FromServices] IOptions<MirrorConfigOptions> options) =>
|
||||
{
|
||||
var config = options.Value;
|
||||
return HttpResults.Ok(new MirrorConfigResponse
|
||||
{
|
||||
Mode = config.Mode,
|
||||
OutputRoot = config.OutputRoot,
|
||||
ConsumerBaseAddress = config.ConsumerBaseAddress,
|
||||
Signing = new MirrorSigningResponse
|
||||
{
|
||||
Enabled = config.SigningEnabled,
|
||||
Algorithm = config.SigningAlgorithm,
|
||||
KeyId = config.SigningKeyId,
|
||||
},
|
||||
AutoRefreshEnabled = config.AutoRefreshEnabled,
|
||||
RefreshIntervalMinutes = config.RefreshIntervalMinutes,
|
||||
});
|
||||
})
|
||||
.WithName("GetMirrorConfig")
|
||||
.WithSummary("Read current mirror configuration")
|
||||
.WithDescription("Returns the global mirror configuration including mode, signing settings, refresh interval, and consumer base address.")
|
||||
.Produces<MirrorConfigResponse>(StatusCodes.Status200OK)
|
||||
.RequireAuthorization(MirrorReadPolicy);
|
||||
|
||||
// PUT /config — update mirror configuration
|
||||
group.MapPut("/config", ([FromBody] UpdateMirrorConfigRequest request, [FromServices] IMirrorConfigStore store, CancellationToken ct) =>
|
||||
{
|
||||
// Note: actual persistence will be implemented with the config store
|
||||
return HttpResults.Ok(new { updated = true });
|
||||
})
|
||||
.WithName("UpdateMirrorConfig")
|
||||
.WithSummary("Update mirror mode, signing, and refresh settings")
|
||||
.WithDescription("Updates the global mirror configuration. Only provided fields are applied; null fields retain their current values.")
|
||||
.Produces(StatusCodes.Status200OK)
|
||||
.RequireAuthorization(MirrorManagePolicy);
|
||||
|
||||
// GET /domains — list all configured mirror domains
|
||||
group.MapGet("/domains", ([FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
|
||||
{
|
||||
var domains = domainStore.GetAllDomains();
|
||||
return HttpResults.Ok(new MirrorDomainListResponse
|
||||
{
|
||||
Domains = domains.Select(MapDomainSummary).ToList(),
|
||||
TotalCount = domains.Count,
|
||||
});
|
||||
})
|
||||
.WithName("ListMirrorDomains")
|
||||
.WithSummary("List all configured mirror domains")
|
||||
.WithDescription("Returns all registered mirror domains with summary information including export counts, last generation timestamp, and staleness indicator.")
|
||||
.Produces<MirrorDomainListResponse>(StatusCodes.Status200OK)
|
||||
.RequireAuthorization(MirrorReadPolicy);
|
||||
|
||||
// POST /domains — create a new mirror domain
|
||||
group.MapPost("/domains", async ([FromBody] CreateMirrorDomainRequest request, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(request.Id) || string.IsNullOrWhiteSpace(request.DisplayName))
|
||||
{
|
||||
return HttpResults.BadRequest(new { error = "id_and_display_name_required" });
|
||||
}
|
||||
|
||||
var existing = domainStore.GetDomain(request.Id);
|
||||
if (existing is not null)
|
||||
{
|
||||
return HttpResults.Conflict(new { error = "domain_already_exists", domainId = request.Id });
|
||||
}
|
||||
|
||||
var domain = new MirrorDomainRecord
|
||||
{
|
||||
Id = request.Id.Trim().ToLowerInvariant(),
|
||||
DisplayName = request.DisplayName.Trim(),
|
||||
RequireAuthentication = request.RequireAuthentication,
|
||||
MaxIndexRequestsPerHour = request.MaxIndexRequestsPerHour ?? 120,
|
||||
MaxDownloadRequestsPerHour = request.MaxDownloadRequestsPerHour ?? 600,
|
||||
Exports = (request.Exports ?? []).Select(e => new MirrorExportRecord
|
||||
{
|
||||
Key = e.Key,
|
||||
Format = e.Format ?? "json",
|
||||
Filters = e.Filters ?? new Dictionary<string, string>(),
|
||||
}).ToList(),
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
|
||||
await domainStore.SaveDomainAsync(domain, ct);
|
||||
|
||||
return HttpResults.Created($"/api/v1/mirror/domains/{domain.Id}", MapDomainDetail(domain));
|
||||
})
|
||||
.WithName("CreateMirrorDomain")
|
||||
.WithSummary("Create a new mirror domain with exports and filters")
|
||||
.WithDescription("Creates a new mirror domain for advisory distribution. The domain ID is normalized to lowercase. Exports define the data slices available for consumption.")
|
||||
.Produces<MirrorDomainDetailResponse>(StatusCodes.Status201Created)
|
||||
.Produces(StatusCodes.Status400BadRequest)
|
||||
.Produces(StatusCodes.Status409Conflict)
|
||||
.RequireAuthorization(MirrorManagePolicy);
|
||||
|
||||
// GET /domains/{domainId} — get domain detail
|
||||
group.MapGet("/domains/{domainId}", ([FromRoute] string domainId, [FromServices] IMirrorDomainStore domainStore) =>
|
||||
{
|
||||
var domain = domainStore.GetDomain(domainId);
|
||||
if (domain is null)
|
||||
{
|
||||
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
|
||||
}
|
||||
|
||||
return HttpResults.Ok(MapDomainDetail(domain));
|
||||
})
|
||||
.WithName("GetMirrorDomain")
|
||||
.WithSummary("Get mirror domain detail with all exports and status")
|
||||
.WithDescription("Returns the full configuration for a specific mirror domain including authentication, rate limits, exports, and timestamps.")
|
||||
.Produces<MirrorDomainDetailResponse>(StatusCodes.Status200OK)
|
||||
.Produces(StatusCodes.Status404NotFound)
|
||||
.RequireAuthorization(MirrorReadPolicy);
|
||||
|
||||
// PUT /domains/{domainId} — update domain
|
||||
group.MapPut("/domains/{domainId}", async ([FromRoute] string domainId, [FromBody] UpdateMirrorDomainRequest request, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
|
||||
{
|
||||
var domain = domainStore.GetDomain(domainId);
|
||||
if (domain is null)
|
||||
{
|
||||
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
|
||||
}
|
||||
|
||||
domain.DisplayName = request.DisplayName ?? domain.DisplayName;
|
||||
domain.RequireAuthentication = request.RequireAuthentication ?? domain.RequireAuthentication;
|
||||
domain.MaxIndexRequestsPerHour = request.MaxIndexRequestsPerHour ?? domain.MaxIndexRequestsPerHour;
|
||||
domain.MaxDownloadRequestsPerHour = request.MaxDownloadRequestsPerHour ?? domain.MaxDownloadRequestsPerHour;
|
||||
|
||||
if (request.Exports is not null)
|
||||
{
|
||||
domain.Exports = request.Exports.Select(e => new MirrorExportRecord
|
||||
{
|
||||
Key = e.Key,
|
||||
Format = e.Format ?? "json",
|
||||
Filters = e.Filters ?? new Dictionary<string, string>(),
|
||||
}).ToList();
|
||||
}
|
||||
|
||||
domain.UpdatedAt = DateTimeOffset.UtcNow;
|
||||
await domainStore.SaveDomainAsync(domain, ct);
|
||||
|
||||
return HttpResults.Ok(MapDomainDetail(domain));
|
||||
})
|
||||
.WithName("UpdateMirrorDomain")
|
||||
.WithSummary("Update mirror domain configuration")
|
||||
.WithDescription("Updates the specified mirror domain. Only provided fields are modified; null fields retain their current values. Providing exports replaces the entire export list.")
|
||||
.Produces<MirrorDomainDetailResponse>(StatusCodes.Status200OK)
|
||||
.Produces(StatusCodes.Status404NotFound)
|
||||
.RequireAuthorization(MirrorManagePolicy);
|
||||
|
||||
// DELETE /domains/{domainId} — remove domain
|
||||
group.MapDelete("/domains/{domainId}", async ([FromRoute] string domainId, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
|
||||
{
|
||||
var domain = domainStore.GetDomain(domainId);
|
||||
if (domain is null)
|
||||
{
|
||||
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
|
||||
}
|
||||
|
||||
await domainStore.DeleteDomainAsync(domainId, ct);
|
||||
return HttpResults.NoContent();
|
||||
})
|
||||
.WithName("DeleteMirrorDomain")
|
||||
.WithSummary("Remove a mirror domain")
|
||||
.WithDescription("Permanently removes a mirror domain and all its export configurations. Active consumers will lose access immediately.")
|
||||
.Produces(StatusCodes.Status204NoContent)
|
||||
.Produces(StatusCodes.Status404NotFound)
|
||||
.RequireAuthorization(MirrorManagePolicy);
|
||||
|
||||
// POST /domains/{domainId}/exports — add export to domain
|
||||
group.MapPost("/domains/{domainId}/exports", async ([FromRoute] string domainId, [FromBody] CreateMirrorExportRequest request, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
|
||||
{
|
||||
var domain = domainStore.GetDomain(domainId);
|
||||
if (domain is null)
|
||||
{
|
||||
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.Key))
|
||||
{
|
||||
return HttpResults.BadRequest(new { error = "export_key_required" });
|
||||
}
|
||||
|
||||
if (domain.Exports.Any(e => e.Key.Equals(request.Key, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
return HttpResults.Conflict(new { error = "export_key_already_exists", key = request.Key });
|
||||
}
|
||||
|
||||
domain.Exports.Add(new MirrorExportRecord
|
||||
{
|
||||
Key = request.Key,
|
||||
Format = request.Format ?? "json",
|
||||
Filters = request.Filters ?? new Dictionary<string, string>(),
|
||||
});
|
||||
domain.UpdatedAt = DateTimeOffset.UtcNow;
|
||||
|
||||
await domainStore.SaveDomainAsync(domain, ct);
|
||||
return HttpResults.Created($"/api/v1/mirror/domains/{domainId}/exports/{request.Key}", new { domainId, exportKey = request.Key });
|
||||
})
|
||||
.WithName("AddMirrorExport")
|
||||
.WithSummary("Add an export to a mirror domain")
|
||||
.WithDescription("Adds a new export definition to the specified mirror domain. Export keys must be unique within a domain. Filters define the advisory subset included in the export.")
|
||||
.Produces(StatusCodes.Status201Created)
|
||||
.Produces(StatusCodes.Status404NotFound)
|
||||
.Produces(StatusCodes.Status409Conflict)
|
||||
.RequireAuthorization(MirrorManagePolicy);
|
||||
|
||||
// DELETE /domains/{domainId}/exports/{exportKey} — remove export
|
||||
group.MapDelete("/domains/{domainId}/exports/{exportKey}", async ([FromRoute] string domainId, [FromRoute] string exportKey, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
|
||||
{
|
||||
var domain = domainStore.GetDomain(domainId);
|
||||
if (domain is null)
|
||||
{
|
||||
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
|
||||
}
|
||||
|
||||
var removed = domain.Exports.RemoveAll(e => e.Key.Equals(exportKey, StringComparison.OrdinalIgnoreCase));
|
||||
if (removed == 0)
|
||||
{
|
||||
return HttpResults.NotFound(new { error = "export_not_found", domainId, exportKey });
|
||||
}
|
||||
|
||||
domain.UpdatedAt = DateTimeOffset.UtcNow;
|
||||
await domainStore.SaveDomainAsync(domain, ct);
|
||||
return HttpResults.NoContent();
|
||||
})
|
||||
.WithName("RemoveMirrorExport")
|
||||
.WithSummary("Remove an export from a mirror domain")
|
||||
.WithDescription("Removes an export definition from the specified mirror domain by key. Returns 404 if the domain or export key does not exist.")
|
||||
.Produces(StatusCodes.Status204NoContent)
|
||||
.Produces(StatusCodes.Status404NotFound)
|
||||
.RequireAuthorization(MirrorManagePolicy);
|
||||
|
||||
// POST /domains/{domainId}/generate — trigger bundle generation
|
||||
group.MapPost("/domains/{domainId}/generate", async ([FromRoute] string domainId, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) =>
|
||||
{
|
||||
var domain = domainStore.GetDomain(domainId);
|
||||
if (domain is null)
|
||||
{
|
||||
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
|
||||
}
|
||||
|
||||
// Mark generation as triggered — actual generation happens async
|
||||
domain.LastGenerateTriggeredAt = DateTimeOffset.UtcNow;
|
||||
await domainStore.SaveDomainAsync(domain, ct);
|
||||
|
||||
return HttpResults.Accepted($"/api/v1/mirror/domains/{domainId}/status", new { domainId, status = "generation_triggered" });
|
||||
})
|
||||
.WithName("TriggerMirrorGeneration")
|
||||
.WithSummary("Trigger bundle generation for a mirror domain")
|
||||
.WithDescription("Triggers asynchronous bundle generation for the specified mirror domain. The generation status can be polled via the domain status endpoint.")
|
||||
.Produces(StatusCodes.Status202Accepted)
|
||||
.Produces(StatusCodes.Status404NotFound)
|
||||
.RequireAuthorization(MirrorManagePolicy);
|
||||
|
||||
// GET /domains/{domainId}/status — get domain status
|
||||
group.MapGet("/domains/{domainId}/status", ([FromRoute] string domainId, [FromServices] IMirrorDomainStore domainStore) =>
|
||||
{
|
||||
var domain = domainStore.GetDomain(domainId);
|
||||
if (domain is null)
|
||||
{
|
||||
return HttpResults.NotFound(new { error = "domain_not_found", domainId });
|
||||
}
|
||||
|
||||
return HttpResults.Ok(new MirrorDomainStatusResponse
|
||||
{
|
||||
DomainId = domain.Id,
|
||||
LastGeneratedAt = domain.LastGeneratedAt,
|
||||
LastGenerateTriggeredAt = domain.LastGenerateTriggeredAt,
|
||||
BundleSizeBytes = domain.BundleSizeBytes,
|
||||
AdvisoryCount = domain.AdvisoryCount,
|
||||
ExportCount = domain.Exports.Count,
|
||||
Staleness = domain.LastGeneratedAt.HasValue
|
||||
? (DateTimeOffset.UtcNow - domain.LastGeneratedAt.Value).TotalMinutes > 120 ? "stale" : "fresh"
|
||||
: "never_generated",
|
||||
});
|
||||
})
|
||||
.WithName("GetMirrorDomainStatus")
|
||||
.WithSummary("Get mirror domain generation status and bundle metrics")
|
||||
.WithDescription("Returns the current generation status for the specified mirror domain including last generation time, bundle size, advisory count, and staleness indicator.")
|
||||
.Produces<MirrorDomainStatusResponse>(StatusCodes.Status200OK)
|
||||
.Produces(StatusCodes.Status404NotFound)
|
||||
.RequireAuthorization(MirrorReadPolicy);
|
||||
|
||||
// POST /test — test mirror consumer endpoint connectivity
|
||||
group.MapPost("/test", async ([FromBody] MirrorTestRequest request, HttpContext httpContext, CancellationToken ct) =>
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(request.BaseAddress))
|
||||
{
|
||||
return HttpResults.BadRequest(new { error = "base_address_required" });
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var httpClientFactory = httpContext.RequestServices.GetRequiredService<IHttpClientFactory>();
|
||||
var client = httpClientFactory.CreateClient("MirrorTest");
|
||||
client.Timeout = TimeSpan.FromSeconds(10);
|
||||
|
||||
var response = await client.GetAsync(
|
||||
$"{request.BaseAddress.TrimEnd('/')}/domains",
|
||||
HttpCompletionOption.ResponseHeadersRead,
|
||||
ct);
|
||||
|
||||
return HttpResults.Ok(new MirrorTestResponse
|
||||
{
|
||||
Reachable = response.IsSuccessStatusCode,
|
||||
StatusCode = (int)response.StatusCode,
|
||||
Message = response.IsSuccessStatusCode ? "Mirror endpoint is reachable" : $"Mirror returned {response.StatusCode}",
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return HttpResults.Ok(new MirrorTestResponse
|
||||
{
|
||||
Reachable = false,
|
||||
Message = $"Connection failed: {ex.Message}",
|
||||
});
|
||||
}
|
||||
})
|
||||
.WithName("TestMirrorEndpoint")
|
||||
.WithSummary("Test mirror consumer endpoint connectivity")
|
||||
.WithDescription("Sends a probe request to the specified mirror consumer base address and reports reachability, HTTP status code, and any connection errors.")
|
||||
.Produces<MirrorTestResponse>(StatusCodes.Status200OK)
|
||||
.RequireAuthorization(MirrorManagePolicy);
|
||||
}
|
||||
|
||||
private static MirrorDomainSummary MapDomainSummary(MirrorDomainRecord domain) => new()
|
||||
{
|
||||
Id = domain.Id,
|
||||
DisplayName = domain.DisplayName,
|
||||
ExportCount = domain.Exports.Count,
|
||||
LastGeneratedAt = domain.LastGeneratedAt,
|
||||
Staleness = domain.LastGeneratedAt.HasValue
|
||||
? (DateTimeOffset.UtcNow - domain.LastGeneratedAt.Value).TotalMinutes > 120 ? "stale" : "fresh"
|
||||
: "never_generated",
|
||||
};
|
||||
|
||||
private static MirrorDomainDetailResponse MapDomainDetail(MirrorDomainRecord domain) => new()
|
||||
{
|
||||
Id = domain.Id,
|
||||
DisplayName = domain.DisplayName,
|
||||
RequireAuthentication = domain.RequireAuthentication,
|
||||
MaxIndexRequestsPerHour = domain.MaxIndexRequestsPerHour,
|
||||
MaxDownloadRequestsPerHour = domain.MaxDownloadRequestsPerHour,
|
||||
Exports = domain.Exports.Select(e => new MirrorExportSummary
|
||||
{
|
||||
Key = e.Key,
|
||||
Format = e.Format,
|
||||
Filters = e.Filters,
|
||||
}).ToList(),
|
||||
LastGeneratedAt = domain.LastGeneratedAt,
|
||||
CreatedAt = domain.CreatedAt,
|
||||
UpdatedAt = domain.UpdatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
// ===== Interfaces =====
|
||||
|
||||
/// <summary>
|
||||
/// Store for mirror domain configuration. Initial implementation: in-memory.
|
||||
/// Future: DB-backed with migration.
|
||||
/// </summary>
|
||||
public interface IMirrorDomainStore
|
||||
{
|
||||
IReadOnlyList<MirrorDomainRecord> GetAllDomains();
|
||||
MirrorDomainRecord? GetDomain(string domainId);
|
||||
Task SaveDomainAsync(MirrorDomainRecord domain, CancellationToken ct = default);
|
||||
Task DeleteDomainAsync(string domainId, CancellationToken ct = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Store for mirror global configuration.
|
||||
/// </summary>
|
||||
public interface IMirrorConfigStore
|
||||
{
|
||||
Task UpdateConfigAsync(UpdateMirrorConfigRequest request, CancellationToken ct = default);
|
||||
}
|
||||
|
||||
// ===== Models =====
|
||||
|
||||
public sealed class MirrorDomainRecord
|
||||
{
|
||||
public required string Id { get; set; }
|
||||
public required string DisplayName { get; set; }
|
||||
public bool RequireAuthentication { get; set; }
|
||||
public int MaxIndexRequestsPerHour { get; set; } = 120;
|
||||
public int MaxDownloadRequestsPerHour { get; set; } = 600;
|
||||
public List<MirrorExportRecord> Exports { get; set; } = [];
|
||||
public DateTimeOffset CreatedAt { get; set; }
|
||||
public DateTimeOffset? UpdatedAt { get; set; }
|
||||
public DateTimeOffset? LastGeneratedAt { get; set; }
|
||||
public DateTimeOffset? LastGenerateTriggeredAt { get; set; }
|
||||
public long BundleSizeBytes { get; set; }
|
||||
public long AdvisoryCount { get; set; }
|
||||
}
|
||||
|
||||
public sealed class MirrorExportRecord
|
||||
{
|
||||
public required string Key { get; set; }
|
||||
public string Format { get; set; } = "json";
|
||||
public Dictionary<string, string> Filters { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class MirrorConfigOptions
|
||||
{
|
||||
public string Mode { get; set; } = "direct";
|
||||
public string? OutputRoot { get; set; }
|
||||
public string? ConsumerBaseAddress { get; set; }
|
||||
public bool SigningEnabled { get; set; }
|
||||
public string? SigningAlgorithm { get; set; }
|
||||
public string? SigningKeyId { get; set; }
|
||||
public bool AutoRefreshEnabled { get; set; } = true;
|
||||
public int RefreshIntervalMinutes { get; set; } = 60;
|
||||
}
|
||||
|
||||
// ===== Request DTOs =====
|
||||
|
||||
public sealed record UpdateMirrorConfigRequest
|
||||
{
|
||||
public string? Mode { get; init; }
|
||||
public string? ConsumerBaseAddress { get; init; }
|
||||
public MirrorSigningRequest? Signing { get; init; }
|
||||
public bool? AutoRefreshEnabled { get; init; }
|
||||
public int? RefreshIntervalMinutes { get; init; }
|
||||
}
|
||||
|
||||
public sealed record MirrorSigningRequest
|
||||
{
|
||||
public bool Enabled { get; init; }
|
||||
public string? Algorithm { get; init; }
|
||||
public string? KeyId { get; init; }
|
||||
}
|
||||
|
||||
public sealed record CreateMirrorDomainRequest
|
||||
{
|
||||
public required string Id { get; init; }
|
||||
public required string DisplayName { get; init; }
|
||||
public bool RequireAuthentication { get; init; }
|
||||
public int? MaxIndexRequestsPerHour { get; init; }
|
||||
public int? MaxDownloadRequestsPerHour { get; init; }
|
||||
public List<CreateMirrorExportRequest>? Exports { get; init; }
|
||||
}
|
||||
|
||||
public sealed record UpdateMirrorDomainRequest
|
||||
{
|
||||
public string? DisplayName { get; init; }
|
||||
public bool? RequireAuthentication { get; init; }
|
||||
public int? MaxIndexRequestsPerHour { get; init; }
|
||||
public int? MaxDownloadRequestsPerHour { get; init; }
|
||||
public List<CreateMirrorExportRequest>? Exports { get; init; }
|
||||
}
|
||||
|
||||
public sealed record CreateMirrorExportRequest
|
||||
{
|
||||
public required string Key { get; init; }
|
||||
public string? Format { get; init; }
|
||||
public Dictionary<string, string>? Filters { get; init; }
|
||||
}
|
||||
|
||||
public sealed record MirrorTestRequest
|
||||
{
|
||||
public required string BaseAddress { get; init; }
|
||||
}
|
||||
|
||||
// ===== Response DTOs =====
|
||||
|
||||
public sealed record MirrorConfigResponse
|
||||
{
|
||||
public string Mode { get; init; } = "direct";
|
||||
public string? OutputRoot { get; init; }
|
||||
public string? ConsumerBaseAddress { get; init; }
|
||||
public MirrorSigningResponse? Signing { get; init; }
|
||||
public bool AutoRefreshEnabled { get; init; }
|
||||
public int RefreshIntervalMinutes { get; init; }
|
||||
}
|
||||
|
||||
public sealed record MirrorSigningResponse
|
||||
{
|
||||
public bool Enabled { get; init; }
|
||||
public string? Algorithm { get; init; }
|
||||
public string? KeyId { get; init; }
|
||||
}
|
||||
|
||||
public sealed record MirrorDomainListResponse
|
||||
{
|
||||
public IReadOnlyList<MirrorDomainSummary> Domains { get; init; } = [];
|
||||
public int TotalCount { get; init; }
|
||||
}
|
||||
|
||||
public sealed record MirrorDomainSummary
|
||||
{
|
||||
public string Id { get; init; } = "";
|
||||
public string DisplayName { get; init; } = "";
|
||||
public int ExportCount { get; init; }
|
||||
public DateTimeOffset? LastGeneratedAt { get; init; }
|
||||
public string Staleness { get; init; } = "never_generated";
|
||||
}
|
||||
|
||||
public sealed record MirrorDomainDetailResponse
|
||||
{
|
||||
public string Id { get; init; } = "";
|
||||
public string DisplayName { get; init; } = "";
|
||||
public bool RequireAuthentication { get; init; }
|
||||
public int MaxIndexRequestsPerHour { get; init; }
|
||||
public int MaxDownloadRequestsPerHour { get; init; }
|
||||
public IReadOnlyList<MirrorExportSummary> Exports { get; init; } = [];
|
||||
public DateTimeOffset? LastGeneratedAt { get; init; }
|
||||
public DateTimeOffset CreatedAt { get; init; }
|
||||
public DateTimeOffset? UpdatedAt { get; init; }
|
||||
}
|
||||
|
||||
public sealed record MirrorExportSummary
|
||||
{
|
||||
public string Key { get; init; } = "";
|
||||
public string Format { get; init; } = "json";
|
||||
public Dictionary<string, string> Filters { get; init; } = new();
|
||||
}
|
||||
|
||||
public sealed record MirrorDomainStatusResponse
|
||||
{
|
||||
public string DomainId { get; init; } = "";
|
||||
public DateTimeOffset? LastGeneratedAt { get; init; }
|
||||
public DateTimeOffset? LastGenerateTriggeredAt { get; init; }
|
||||
public long BundleSizeBytes { get; init; }
|
||||
public long AdvisoryCount { get; init; }
|
||||
public int ExportCount { get; init; }
|
||||
public string Staleness { get; init; } = "never_generated";
|
||||
}
|
||||
|
||||
public sealed record MirrorTestResponse
|
||||
{
|
||||
public bool Reachable { get; init; }
|
||||
public int? StatusCode { get; init; }
|
||||
public string? Message { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
using HttpResults = Microsoft.AspNetCore.Http.Results;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using StellaOps.Auth.ServerIntegration.Tenancy;
|
||||
using StellaOps.Concelier.Core.Sources;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Extensions;
|
||||
|
||||
/// <summary>
|
||||
/// Source management endpoints for the advisory source registry.
|
||||
/// Provides catalog browsing, connectivity checks, and enable/disable controls.
|
||||
/// </summary>
|
||||
internal static class SourceManagementEndpointExtensions
|
||||
{
|
||||
private const string AdvisoryReadPolicy = "Concelier.Advisories.Read";
|
||||
private const string SourcesManagePolicy = "Concelier.Sources.Manage";
|
||||
|
||||
public static void MapSourceManagementEndpoints(this WebApplication app)
|
||||
{
|
||||
var group = app.MapGroup("/api/v1/sources")
|
||||
.WithTags("Source Management")
|
||||
.RequireTenant();
|
||||
|
||||
// GET /catalog — list all registered source definitions
|
||||
group.MapGet("/catalog", (
|
||||
[FromServices] ISourceRegistry registry) =>
|
||||
{
|
||||
var sources = registry.GetAllSources();
|
||||
var items = sources.Select(MapCatalogItem).ToList();
|
||||
|
||||
return HttpResults.Ok(new SourceCatalogResponse
|
||||
{
|
||||
Items = items,
|
||||
TotalCount = items.Count
|
||||
});
|
||||
})
|
||||
.WithName("GetSourceCatalog")
|
||||
.WithSummary("List all registered advisory source definitions")
|
||||
.WithDescription("Returns the full catalog of advisory data sources with their configuration, endpoints, and default settings.")
|
||||
.Produces<SourceCatalogResponse>(StatusCodes.Status200OK)
|
||||
.RequireAuthorization(AdvisoryReadPolicy);
|
||||
|
||||
// GET /status — enabled sources with last check results
|
||||
group.MapGet("/status", async (
|
||||
[FromServices] ISourceRegistry registry,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var enabledIds = await registry.GetEnabledSourcesAsync(cancellationToken).ConfigureAwait(false);
|
||||
var allSources = registry.GetAllSources();
|
||||
var items = new List<SourceStatusItem>(allSources.Count);
|
||||
|
||||
foreach (var source in allSources)
|
||||
{
|
||||
items.Add(new SourceStatusItem
|
||||
{
|
||||
SourceId = source.Id,
|
||||
Enabled = enabledIds.Contains(source.Id, StringComparer.OrdinalIgnoreCase),
|
||||
LastCheck = registry.GetLastCheckResult(source.Id)
|
||||
});
|
||||
}
|
||||
|
||||
return HttpResults.Ok(new SourceStatusResponse { Sources = items });
|
||||
})
|
||||
.WithName("GetSourceStatus")
|
||||
.WithSummary("Get status of all sources with last connectivity check")
|
||||
.WithDescription("Returns enabled/disabled state and last connectivity check result for every registered source.")
|
||||
.Produces<SourceStatusResponse>(StatusCodes.Status200OK)
|
||||
.RequireAuthorization(AdvisoryReadPolicy);
|
||||
|
||||
// POST /{sourceId}/enable — enable a single source
|
||||
group.MapPost("/{sourceId}/enable", async (
|
||||
string sourceId,
|
||||
[FromServices] ISourceRegistry registry,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var source = registry.GetSource(sourceId);
|
||||
if (source is null)
|
||||
{
|
||||
return HttpResults.NotFound(new { error = "source_not_found", sourceId });
|
||||
}
|
||||
|
||||
var success = await registry.EnableSourceAsync(sourceId, cancellationToken).ConfigureAwait(false);
|
||||
return success
|
||||
? HttpResults.Ok(new { sourceId, enabled = true })
|
||||
: HttpResults.UnprocessableEntity(new { error = "enable_failed", sourceId });
|
||||
})
|
||||
.WithName("EnableSource")
|
||||
.WithSummary("Enable a source for data ingestion")
|
||||
.WithDescription("Enables the specified advisory source so it will be included in data ingestion runs.")
|
||||
.Produces(StatusCodes.Status200OK)
|
||||
.Produces(StatusCodes.Status404NotFound)
|
||||
.Produces(StatusCodes.Status422UnprocessableEntity)
|
||||
.RequireAuthorization(SourcesManagePolicy);
|
||||
|
||||
// POST /{sourceId}/disable — disable a single source
|
||||
group.MapPost("/{sourceId}/disable", async (
|
||||
string sourceId,
|
||||
[FromServices] ISourceRegistry registry,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var source = registry.GetSource(sourceId);
|
||||
if (source is null)
|
||||
{
|
||||
return HttpResults.NotFound(new { error = "source_not_found", sourceId });
|
||||
}
|
||||
|
||||
var success = await registry.DisableSourceAsync(sourceId, cancellationToken).ConfigureAwait(false);
|
||||
return success
|
||||
? HttpResults.Ok(new { sourceId, enabled = false })
|
||||
: HttpResults.UnprocessableEntity(new { error = "disable_failed", sourceId });
|
||||
})
|
||||
.WithName("DisableSource")
|
||||
.WithSummary("Disable a source")
|
||||
.WithDescription("Disables the specified advisory source so it will be excluded from data ingestion runs.")
|
||||
.Produces(StatusCodes.Status200OK)
|
||||
.Produces(StatusCodes.Status404NotFound)
|
||||
.Produces(StatusCodes.Status422UnprocessableEntity)
|
||||
.RequireAuthorization(SourcesManagePolicy);
|
||||
|
||||
// POST /check — check all sources and auto-configure
|
||||
group.MapPost("/check", async (
|
||||
[FromServices] ISourceRegistry registry,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var result = await registry.CheckAllAndAutoConfigureAsync(cancellationToken).ConfigureAwait(false);
|
||||
return HttpResults.Ok(result);
|
||||
})
|
||||
.WithName("CheckAllSources")
|
||||
.WithSummary("Check connectivity for all sources and auto-configure")
|
||||
.WithDescription("Runs connectivity checks against all registered sources. Healthy sources are auto-enabled; failed sources are disabled.")
|
||||
.Produces<SourceCheckResult>(StatusCodes.Status200OK)
|
||||
.RequireAuthorization(SourcesManagePolicy);
|
||||
|
||||
// POST /{sourceId}/check — check connectivity for a single source
|
||||
group.MapPost("/{sourceId}/check", async (
|
||||
string sourceId,
|
||||
[FromServices] ISourceRegistry registry,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var source = registry.GetSource(sourceId);
|
||||
if (source is null)
|
||||
{
|
||||
return HttpResults.NotFound(new { error = "source_not_found", sourceId });
|
||||
}
|
||||
|
||||
var result = await registry.CheckConnectivityAsync(sourceId, cancellationToken).ConfigureAwait(false);
|
||||
return HttpResults.Ok(result);
|
||||
})
|
||||
.WithName("CheckSourceConnectivity")
|
||||
.WithSummary("Check connectivity for a single source")
|
||||
.WithDescription("Runs a connectivity check against the specified source and returns detailed status, latency, and remediation steps if failed.")
|
||||
.Produces<SourceConnectivityResult>(StatusCodes.Status200OK)
|
||||
.Produces(StatusCodes.Status404NotFound)
|
||||
.RequireAuthorization(SourcesManagePolicy);
|
||||
|
||||
// POST /batch-enable — enable multiple sources
|
||||
group.MapPost("/batch-enable", async (
|
||||
[FromBody] BatchSourceRequest request,
|
||||
[FromServices] ISourceRegistry registry,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (request.SourceIds is null || request.SourceIds.Count == 0)
|
||||
{
|
||||
return HttpResults.BadRequest(new { error = "source_ids_required" });
|
||||
}
|
||||
|
||||
var results = new List<BatchSourceResultItem>(request.SourceIds.Count);
|
||||
foreach (var id in request.SourceIds)
|
||||
{
|
||||
var source = registry.GetSource(id);
|
||||
if (source is null)
|
||||
{
|
||||
results.Add(new BatchSourceResultItem { SourceId = id, Success = false, Error = "source_not_found" });
|
||||
continue;
|
||||
}
|
||||
|
||||
var success = await registry.EnableSourceAsync(id, cancellationToken).ConfigureAwait(false);
|
||||
results.Add(new BatchSourceResultItem { SourceId = id, Success = success, Error = success ? null : "enable_failed" });
|
||||
}
|
||||
|
||||
return HttpResults.Ok(new BatchSourceResponse { Results = results });
|
||||
})
|
||||
.WithName("BatchEnableSources")
|
||||
.WithSummary("Enable multiple sources in a single request")
|
||||
.WithDescription("Enables each specified source for data ingestion. Returns per-source success/failure results.")
|
||||
.Produces<BatchSourceResponse>(StatusCodes.Status200OK)
|
||||
.Produces(StatusCodes.Status400BadRequest)
|
||||
.RequireAuthorization(SourcesManagePolicy);
|
||||
|
||||
// POST /batch-disable — disable multiple sources
|
||||
group.MapPost("/batch-disable", async (
|
||||
[FromBody] BatchSourceRequest request,
|
||||
[FromServices] ISourceRegistry registry,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (request.SourceIds is null || request.SourceIds.Count == 0)
|
||||
{
|
||||
return HttpResults.BadRequest(new { error = "source_ids_required" });
|
||||
}
|
||||
|
||||
var results = new List<BatchSourceResultItem>(request.SourceIds.Count);
|
||||
foreach (var id in request.SourceIds)
|
||||
{
|
||||
var source = registry.GetSource(id);
|
||||
if (source is null)
|
||||
{
|
||||
results.Add(new BatchSourceResultItem { SourceId = id, Success = false, Error = "source_not_found" });
|
||||
continue;
|
||||
}
|
||||
|
||||
var success = await registry.DisableSourceAsync(id, cancellationToken).ConfigureAwait(false);
|
||||
results.Add(new BatchSourceResultItem { SourceId = id, Success = success, Error = success ? null : "disable_failed" });
|
||||
}
|
||||
|
||||
return HttpResults.Ok(new BatchSourceResponse { Results = results });
|
||||
})
|
||||
.WithName("BatchDisableSources")
|
||||
.WithSummary("Disable multiple sources in a single request")
|
||||
.WithDescription("Disables each specified source. Returns per-source success/failure results.")
|
||||
.Produces<BatchSourceResponse>(StatusCodes.Status200OK)
|
||||
.Produces(StatusCodes.Status400BadRequest)
|
||||
.RequireAuthorization(SourcesManagePolicy);
|
||||
|
||||
// GET /{sourceId}/check-result — get last check result for a source
|
||||
group.MapGet("/{sourceId}/check-result", (
|
||||
string sourceId,
|
||||
[FromServices] ISourceRegistry registry) =>
|
||||
{
|
||||
var source = registry.GetSource(sourceId);
|
||||
if (source is null)
|
||||
{
|
||||
return HttpResults.NotFound(new { error = "source_not_found", sourceId });
|
||||
}
|
||||
|
||||
var result = registry.GetLastCheckResult(sourceId);
|
||||
if (result is null)
|
||||
{
|
||||
return HttpResults.Ok(new { sourceId, lastCheck = (SourceConnectivityResult?)null, message = "no_check_performed" });
|
||||
}
|
||||
|
||||
return HttpResults.Ok(result);
|
||||
})
|
||||
.WithName("GetSourceCheckResult")
|
||||
.WithSummary("Get last connectivity check result for a source")
|
||||
.WithDescription("Returns the most recent connectivity check result for the specified source, including status, latency, and any error details.")
|
||||
.Produces<SourceConnectivityResult>(StatusCodes.Status200OK)
|
||||
.Produces(StatusCodes.Status404NotFound)
|
||||
.RequireAuthorization(AdvisoryReadPolicy);
|
||||
}
|
||||
|
||||
private static SourceCatalogItem MapCatalogItem(SourceDefinition source)
|
||||
{
|
||||
return new SourceCatalogItem
|
||||
{
|
||||
Id = source.Id,
|
||||
DisplayName = source.DisplayName,
|
||||
Category = source.Category.ToString(),
|
||||
Type = source.Type.ToString(),
|
||||
Description = source.Description,
|
||||
BaseEndpoint = source.BaseEndpoint,
|
||||
RequiresAuth = source.RequiresAuthentication,
|
||||
CredentialEnvVar = source.CredentialEnvVar,
|
||||
CredentialUrl = source.CredentialUrl,
|
||||
DocumentationUrl = source.DocumentationUrl,
|
||||
DefaultPriority = source.DefaultPriority,
|
||||
Regions = source.Regions,
|
||||
Tags = source.Tags,
|
||||
EnabledByDefault = source.EnabledByDefault
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ===== Response DTOs =====
|
||||
|
||||
public sealed record SourceCatalogResponse
|
||||
{
|
||||
public IReadOnlyList<SourceCatalogItem> Items { get; init; } = [];
|
||||
public int TotalCount { get; init; }
|
||||
}
|
||||
|
||||
public sealed record SourceCatalogItem
|
||||
{
|
||||
public string Id { get; init; } = string.Empty;
|
||||
public string DisplayName { get; init; } = string.Empty;
|
||||
public string Category { get; init; } = string.Empty;
|
||||
public string Type { get; init; } = string.Empty;
|
||||
public string Description { get; init; } = string.Empty;
|
||||
public string BaseEndpoint { get; init; } = string.Empty;
|
||||
public bool RequiresAuth { get; init; }
|
||||
public string? CredentialEnvVar { get; init; }
|
||||
public string? CredentialUrl { get; init; }
|
||||
public string? DocumentationUrl { get; init; }
|
||||
public int DefaultPriority { get; init; }
|
||||
public IReadOnlyList<string> Regions { get; init; } = [];
|
||||
public IReadOnlyList<string> Tags { get; init; } = [];
|
||||
public bool EnabledByDefault { get; init; }
|
||||
}
|
||||
|
||||
public sealed record SourceStatusResponse
|
||||
{
|
||||
public IReadOnlyList<SourceStatusItem> Sources { get; init; } = [];
|
||||
}
|
||||
|
||||
public sealed record SourceStatusItem
|
||||
{
|
||||
public string SourceId { get; init; } = string.Empty;
|
||||
public bool Enabled { get; init; }
|
||||
public SourceConnectivityResult? LastCheck { get; init; }
|
||||
}
|
||||
|
||||
public sealed record BatchSourceRequest
|
||||
{
|
||||
public IReadOnlyList<string> SourceIds { get; init; } = [];
|
||||
}
|
||||
|
||||
public sealed record BatchSourceResponse
|
||||
{
|
||||
public IReadOnlyList<BatchSourceResultItem> Results { get; init; } = [];
|
||||
}
|
||||
|
||||
public sealed record BatchSourceResultItem
|
||||
{
|
||||
public string SourceId { get; init; } = string.Empty;
|
||||
public bool Success { get; init; }
|
||||
public string? Error { get; init; }
|
||||
}
|
||||
@@ -35,6 +35,7 @@ using StellaOps.Concelier.Core.Observations;
|
||||
using StellaOps.Concelier.Core.Orchestration;
|
||||
using StellaOps.Concelier.Core.Raw;
|
||||
using StellaOps.Concelier.Core.Signals;
|
||||
using StellaOps.Concelier.Core.Sources;
|
||||
using StellaOps.Concelier.Merge;
|
||||
using StellaOps.Concelier.Merge.Services;
|
||||
using StellaOps.Concelier.Models;
|
||||
@@ -55,6 +56,7 @@ using StellaOps.Concelier.WebService.Results;
|
||||
using StellaOps.Concelier.WebService.Services;
|
||||
using StellaOps.Concelier.WebService.Telemetry;
|
||||
using StellaOps.Configuration;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Plugin.DependencyInjection;
|
||||
using StellaOps.Plugin.Hosting;
|
||||
using StellaOps.Provenance;
|
||||
@@ -557,6 +559,20 @@ builder.Services.AddConcelierOrchestrationServices();
|
||||
// Register federation snapshot coordination services (SPRINT_20260208_035)
|
||||
builder.Services.AddConcelierFederationServices();
|
||||
|
||||
// Register advisory source registry and connectivity services
|
||||
builder.Services.AddSourcesRegistry(builder.Configuration);
|
||||
|
||||
// Mirror domain management (in-memory store, future: DB-backed)
|
||||
builder.Services.AddSingleton<InMemoryMirrorDomainStore>();
|
||||
builder.Services.AddSingleton<IMirrorDomainStore>(sp => sp.GetRequiredService<InMemoryMirrorDomainStore>());
|
||||
builder.Services.AddSingleton<IMirrorConfigStore>(sp => sp.GetRequiredService<InMemoryMirrorDomainStore>());
|
||||
builder.Services.Configure<MirrorConfigOptions>(builder.Configuration.GetSection("Mirror"));
|
||||
builder.Services.AddHttpClient("MirrorTest");
|
||||
|
||||
// Mirror distribution options binding and export scheduler (background bundle refresh, TASK-006b)
|
||||
builder.Services.Configure<MirrorDistributionOptions>(builder.Configuration.GetSection(MirrorDistributionOptions.SectionName));
|
||||
builder.Services.AddHostedService<MirrorExportScheduler>();
|
||||
|
||||
var features = concelierOptions.Features ?? new ConcelierOptions.FeaturesOptions();
|
||||
|
||||
if (!features.NoMergeEnabled)
|
||||
@@ -931,6 +947,7 @@ app.MapConcelierMirrorEndpoints(authorityConfigured, enforceAuthority);
|
||||
// Canonical advisory endpoints (Sprint 8200.0012.0003)
|
||||
app.MapCanonicalAdvisoryEndpoints();
|
||||
app.MapAdvisorySourceEndpoints();
|
||||
app.MapSourceManagementEndpoints();
|
||||
app.MapInterestScoreEndpoints();
|
||||
|
||||
// Federation endpoints for site-to-site bundle sync
|
||||
@@ -945,6 +962,9 @@ app.MapFeedSnapshotEndpoints();
|
||||
// Feed mirror management, bundles, version locks, offline status
|
||||
app.MapFeedMirrorManagementEndpoints();
|
||||
|
||||
// Mirror domain management CRUD endpoints
|
||||
app.MapMirrorDomainManagementEndpoints();
|
||||
|
||||
app.MapGet("/.well-known/openapi", ([FromServices] OpenApiDiscoveryDocumentProvider provider, HttpContext context) =>
|
||||
{
|
||||
var (payload, etag) = provider.GetDocument();
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
using System.Collections.Concurrent;
|
||||
using StellaOps.Concelier.WebService.Extensions;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Services;
|
||||
|
||||
/// <summary>
|
||||
/// In-memory implementation of <see cref="IMirrorDomainStore"/> and <see cref="IMirrorConfigStore"/>.
|
||||
/// Suitable for development and single-instance deployments. Future: replace with DB-backed store.
|
||||
/// </summary>
|
||||
public sealed class InMemoryMirrorDomainStore : IMirrorDomainStore, IMirrorConfigStore
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, MirrorDomainRecord> _domains = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public IReadOnlyList<MirrorDomainRecord> GetAllDomains() => _domains.Values.ToList();
|
||||
|
||||
public MirrorDomainRecord? GetDomain(string domainId) => _domains.GetValueOrDefault(domainId);
|
||||
|
||||
public Task SaveDomainAsync(MirrorDomainRecord domain, CancellationToken ct = default)
|
||||
{
|
||||
_domains[domain.Id] = domain;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DeleteDomainAsync(string domainId, CancellationToken ct = default)
|
||||
{
|
||||
_domains.TryRemove(domainId, out _);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task UpdateConfigAsync(UpdateMirrorConfigRequest request, CancellationToken ct = default)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,7 @@
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Concelier.Interest/StellaOps.Concelier.Interest.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Concelier.SbomIntegration/StellaOps.Concelier.SbomIntegration.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Concelier.Persistence/StellaOps.Concelier.Persistence.csproj" />
|
||||
|
||||
@@ -128,6 +128,21 @@ public enum SourceCategory
|
||||
/// <summary>Exploit and threat intelligence sources.</summary>
|
||||
Threat,
|
||||
|
||||
/// <summary>Exploit databases and PoC sources.</summary>
|
||||
Exploit,
|
||||
|
||||
/// <summary>Container image advisory sources.</summary>
|
||||
Container,
|
||||
|
||||
/// <summary>Hardware and firmware PSIRT advisories.</summary>
|
||||
Hardware,
|
||||
|
||||
/// <summary>Industrial control systems and SCADA advisories.</summary>
|
||||
Ics,
|
||||
|
||||
/// <summary>Package manager native advisory databases (cargo-audit, pip-audit, govulncheck, bundler-audit).</summary>
|
||||
PackageManager,
|
||||
|
||||
/// <summary>StellaOps mirrors.</summary>
|
||||
Mirror,
|
||||
|
||||
@@ -150,7 +165,10 @@ public enum SourceType
|
||||
LocalFile,
|
||||
|
||||
/// <summary>Custom/user-defined source.</summary>
|
||||
Custom
|
||||
Custom,
|
||||
|
||||
/// <summary>STIX/TAXII protocol feed.</summary>
|
||||
StixTaxii
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -894,6 +912,440 @@ public static class SourceDefinitions
|
||||
Tags = ImmutableArray.Create("cert", "us", "cisa")
|
||||
};
|
||||
|
||||
// ===== Exploit Databases =====
|
||||
|
||||
public static readonly SourceDefinition ExploitDb = new()
|
||||
{
|
||||
Id = "exploitdb",
|
||||
DisplayName = "Exploit-DB",
|
||||
Category = SourceCategory.Exploit,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Offensive Security Exploit Database",
|
||||
BaseEndpoint = "https://gitlab.com/exploit-database/exploitdb/-/raw/main/",
|
||||
HealthCheckEndpoint = "https://gitlab.com/exploit-database/exploitdb",
|
||||
HttpClientName = "ExploitDbClient",
|
||||
RequiresAuthentication = false,
|
||||
DocumentationUrl = "https://www.exploit-db.com/",
|
||||
DefaultPriority = 110,
|
||||
Tags = ImmutableArray.Create("exploit", "poc", "offensive")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition PocGithub = new()
|
||||
{
|
||||
Id = "poc-github",
|
||||
DisplayName = "PoC-in-GitHub",
|
||||
Category = SourceCategory.Exploit,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "GitHub repositories containing vulnerability PoCs",
|
||||
BaseEndpoint = "https://api.github.com/search/repositories",
|
||||
HealthCheckEndpoint = "https://api.github.com/zen",
|
||||
HttpClientName = "PocGithubClient",
|
||||
RequiresAuthentication = true,
|
||||
CredentialEnvVar = "GITHUB_PAT",
|
||||
DefaultPriority = 112,
|
||||
Tags = ImmutableArray.Create("exploit", "poc", "github")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition Metasploit = new()
|
||||
{
|
||||
Id = "metasploit",
|
||||
DisplayName = "Metasploit Modules",
|
||||
Category = SourceCategory.Exploit,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Rapid7 Metasploit Framework vulnerability modules",
|
||||
BaseEndpoint = "https://raw.githubusercontent.com/rapid7/metasploit-framework/master/",
|
||||
HealthCheckEndpoint = "https://raw.githubusercontent.com/rapid7/metasploit-framework/master/README.md",
|
||||
HttpClientName = "MetasploitClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 114,
|
||||
Tags = ImmutableArray.Create("exploit", "metasploit", "rapid7")
|
||||
};
|
||||
|
||||
// ===== Cloud Provider Advisories =====
|
||||
|
||||
public static readonly SourceDefinition Aws = new()
|
||||
{
|
||||
Id = "aws",
|
||||
DisplayName = "AWS Security Bulletins",
|
||||
Category = SourceCategory.Vendor,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Amazon Web Services security bulletins",
|
||||
BaseEndpoint = "https://aws.amazon.com/security/security-bulletins/",
|
||||
HealthCheckEndpoint = "https://aws.amazon.com/security/security-bulletins/",
|
||||
HttpClientName = "AwsClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 81,
|
||||
Tags = ImmutableArray.Create("aws", "vendor", "cloud")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition Azure = new()
|
||||
{
|
||||
Id = "azure",
|
||||
DisplayName = "Azure Security Advisories",
|
||||
Category = SourceCategory.Vendor,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Microsoft Azure security advisories",
|
||||
BaseEndpoint = "https://api.msrc.microsoft.com/sug/v2.0/en-US/",
|
||||
HealthCheckEndpoint = "https://api.msrc.microsoft.com/sug/v2.0/en-US/affectedProduct",
|
||||
HttpClientName = "AzureClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 82,
|
||||
Tags = ImmutableArray.Create("azure", "vendor", "cloud", "microsoft")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition Gcp = new()
|
||||
{
|
||||
Id = "gcp",
|
||||
DisplayName = "GCP Security Bulletins",
|
||||
Category = SourceCategory.Vendor,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Google Cloud Platform security bulletins",
|
||||
BaseEndpoint = "https://cloud.google.com/support/bulletins/",
|
||||
HealthCheckEndpoint = "https://cloud.google.com/support/bulletins/",
|
||||
HttpClientName = "GcpClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 83,
|
||||
Tags = ImmutableArray.Create("gcp", "vendor", "cloud", "google")
|
||||
};
|
||||
|
||||
// ===== Container Sources =====
|
||||
|
||||
public static readonly SourceDefinition DockerOfficial = new()
|
||||
{
|
||||
Id = "docker-official",
|
||||
DisplayName = "Docker Official CVEs",
|
||||
Category = SourceCategory.Container,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Docker Official Images CVE notices",
|
||||
BaseEndpoint = "https://hub.docker.com/v2/",
|
||||
HealthCheckEndpoint = "https://hub.docker.com/v2/",
|
||||
HttpClientName = "DockerOfficialClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 120,
|
||||
Tags = ImmutableArray.Create("docker", "container", "oci")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition Chainguard = new()
|
||||
{
|
||||
Id = "chainguard",
|
||||
DisplayName = "Chainguard Advisories",
|
||||
Category = SourceCategory.Container,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Chainguard hardened image advisories",
|
||||
BaseEndpoint = "https://images.chainguard.dev/",
|
||||
HealthCheckEndpoint = "https://images.chainguard.dev/",
|
||||
HttpClientName = "ChainguardClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 122,
|
||||
Tags = ImmutableArray.Create("chainguard", "container", "hardened")
|
||||
};
|
||||
|
||||
// ===== Hardware/Firmware =====
|
||||
|
||||
public static readonly SourceDefinition Intel = new()
|
||||
{
|
||||
Id = "intel",
|
||||
DisplayName = "Intel PSIRT",
|
||||
Category = SourceCategory.Hardware,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Intel Product Security Incident Response Team",
|
||||
BaseEndpoint = "https://www.intel.com/content/www/us/en/security-center/default.html",
|
||||
HealthCheckEndpoint = "https://www.intel.com/content/www/us/en/security-center/default.html",
|
||||
HttpClientName = "IntelClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 130,
|
||||
Tags = ImmutableArray.Create("intel", "hardware", "firmware", "cpu")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition Amd = new()
|
||||
{
|
||||
Id = "amd",
|
||||
DisplayName = "AMD Security",
|
||||
Category = SourceCategory.Hardware,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "AMD Product Security advisories",
|
||||
BaseEndpoint = "https://www.amd.com/en/resources/product-security.html",
|
||||
HealthCheckEndpoint = "https://www.amd.com/en/resources/product-security.html",
|
||||
HttpClientName = "AmdClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 132,
|
||||
Tags = ImmutableArray.Create("amd", "hardware", "firmware", "cpu")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition Arm = new()
|
||||
{
|
||||
Id = "arm",
|
||||
DisplayName = "ARM Security Center",
|
||||
Category = SourceCategory.Hardware,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "ARM Security Center advisories",
|
||||
BaseEndpoint = "https://developer.arm.com/Arm%20Security%20Center/",
|
||||
HealthCheckEndpoint = "https://developer.arm.com/Arm%20Security%20Center/",
|
||||
HttpClientName = "ArmClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 134,
|
||||
Tags = ImmutableArray.Create("arm", "hardware", "firmware", "cpu")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition Siemens = new()
|
||||
{
|
||||
Id = "siemens",
|
||||
DisplayName = "Siemens ProductCERT",
|
||||
Category = SourceCategory.Ics,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Siemens Product CERT ICS advisories",
|
||||
BaseEndpoint = "https://cert-portal.siemens.com/productcert/csaf/",
|
||||
HealthCheckEndpoint = "https://cert-portal.siemens.com/productcert/",
|
||||
HttpClientName = "SiemensClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 136,
|
||||
Tags = ImmutableArray.Create("siemens", "ics", "scada", "hardware")
|
||||
};
|
||||
|
||||
// ===== Package Manager Native Advisories =====
|
||||
|
||||
public static readonly SourceDefinition RustSec = new()
|
||||
{
|
||||
Id = "rustsec",
|
||||
DisplayName = "RustSec Advisory DB",
|
||||
Category = SourceCategory.PackageManager,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Rust Security Advisory Database (cargo-audit)",
|
||||
BaseEndpoint = "https://raw.githubusercontent.com/rustsec/advisory-db/main/",
|
||||
HealthCheckEndpoint = "https://raw.githubusercontent.com/rustsec/advisory-db/main/README.md",
|
||||
HttpClientName = "RustSecClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 63,
|
||||
Tags = ImmutableArray.Create("rustsec", "package-manager", "rust", "cargo")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition PyPa = new()
|
||||
{
|
||||
Id = "pypa",
|
||||
DisplayName = "PyPA Advisory DB",
|
||||
Category = SourceCategory.PackageManager,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Python Packaging Authority Advisory Database (pip-audit)",
|
||||
BaseEndpoint = "https://raw.githubusercontent.com/pypa/advisory-database/main/",
|
||||
HealthCheckEndpoint = "https://raw.githubusercontent.com/pypa/advisory-database/main/README.md",
|
||||
HttpClientName = "PyPaClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 53,
|
||||
Tags = ImmutableArray.Create("pypa", "package-manager", "python", "pip")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition GoVuln = new()
|
||||
{
|
||||
Id = "govuln",
|
||||
DisplayName = "Go Vuln DB",
|
||||
Category = SourceCategory.PackageManager,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Go Vulnerability Database (govulncheck)",
|
||||
BaseEndpoint = "https://vuln.go.dev/",
|
||||
HealthCheckEndpoint = "https://vuln.go.dev/",
|
||||
HttpClientName = "GoVulnClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 55,
|
||||
Tags = ImmutableArray.Create("govuln", "package-manager", "go", "golang")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition BundlerAudit = new()
|
||||
{
|
||||
Id = "bundler-audit",
|
||||
DisplayName = "Ruby Advisory DB",
|
||||
Category = SourceCategory.PackageManager,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Ruby Advisory Database (bundler-audit)",
|
||||
BaseEndpoint = "https://raw.githubusercontent.com/rubysec/ruby-advisory-db/main/",
|
||||
HealthCheckEndpoint = "https://raw.githubusercontent.com/rubysec/ruby-advisory-db/main/README.md",
|
||||
HttpClientName = "BundlerAuditClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 57,
|
||||
Tags = ImmutableArray.Create("bundler", "package-manager", "ruby", "rubysec")
|
||||
};
|
||||
|
||||
// ===== Additional CERTs =====
|
||||
|
||||
public static readonly SourceDefinition CertUa = new()
|
||||
{
|
||||
Id = "cert-ua",
|
||||
DisplayName = "CERT-UA (Ukraine)",
|
||||
Category = SourceCategory.Cert,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Ukrainian Computer Emergency Response Team",
|
||||
BaseEndpoint = "https://cert.gov.ua/",
|
||||
HealthCheckEndpoint = "https://cert.gov.ua/",
|
||||
HttpClientName = "CertUaClient",
|
||||
RequiresAuthentication = false,
|
||||
Regions = ImmutableArray.Create("UA"),
|
||||
DefaultPriority = 95,
|
||||
Tags = ImmutableArray.Create("cert", "ukraine")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition CertPl = new()
|
||||
{
|
||||
Id = "cert-pl",
|
||||
DisplayName = "CERT.PL (Poland)",
|
||||
Category = SourceCategory.Cert,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Polish CERT",
|
||||
BaseEndpoint = "https://cert.pl/en/",
|
||||
HealthCheckEndpoint = "https://cert.pl/en/",
|
||||
HttpClientName = "CertPlClient",
|
||||
RequiresAuthentication = false,
|
||||
Regions = ImmutableArray.Create("PL", "EU"),
|
||||
DefaultPriority = 96,
|
||||
Tags = ImmutableArray.Create("cert", "poland", "eu")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition AusCert = new()
|
||||
{
|
||||
Id = "auscert",
|
||||
DisplayName = "AusCERT (Australia)",
|
||||
Category = SourceCategory.Cert,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Australian Cyber Security Centre CERT",
|
||||
BaseEndpoint = "https://auscert.org.au/",
|
||||
HealthCheckEndpoint = "https://auscert.org.au/",
|
||||
HttpClientName = "AusCertClient",
|
||||
RequiresAuthentication = false,
|
||||
Regions = ImmutableArray.Create("AU", "APAC"),
|
||||
DefaultPriority = 97,
|
||||
Tags = ImmutableArray.Create("cert", "australia", "apac")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition KrCert = new()
|
||||
{
|
||||
Id = "krcert",
|
||||
DisplayName = "KrCERT/CC (South Korea)",
|
||||
Category = SourceCategory.Cert,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Korean Computer Emergency Response Team",
|
||||
BaseEndpoint = "https://www.krcert.or.kr/",
|
||||
HealthCheckEndpoint = "https://www.krcert.or.kr/",
|
||||
HttpClientName = "KrCertClient",
|
||||
RequiresAuthentication = false,
|
||||
Regions = ImmutableArray.Create("KR", "APAC"),
|
||||
DefaultPriority = 98,
|
||||
Tags = ImmutableArray.Create("cert", "korea", "apac")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition CertIn = new()
|
||||
{
|
||||
Id = "cert-in",
|
||||
DisplayName = "CERT-In (India)",
|
||||
Category = SourceCategory.Cert,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Indian Computer Emergency Response Team",
|
||||
BaseEndpoint = "https://www.cert-in.org.in/",
|
||||
HealthCheckEndpoint = "https://www.cert-in.org.in/",
|
||||
HttpClientName = "CertInClient",
|
||||
RequiresAuthentication = false,
|
||||
Regions = ImmutableArray.Create("IN", "APAC"),
|
||||
DefaultPriority = 99,
|
||||
Tags = ImmutableArray.Create("cert", "india", "apac")
|
||||
};
|
||||
|
||||
// ===== Russian/CIS Sources =====
|
||||
|
||||
public static readonly SourceDefinition FstecBdu = new()
|
||||
{
|
||||
Id = "fstec-bdu",
|
||||
DisplayName = "FSTEC BDU (Russia)",
|
||||
Category = SourceCategory.Cert,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Federal Service for Technical and Export Control — Bank of Security Threats",
|
||||
BaseEndpoint = "https://bdu.fstec.ru/",
|
||||
HealthCheckEndpoint = "https://bdu.fstec.ru/",
|
||||
HttpClientName = "FstecBduClient",
|
||||
RequiresAuthentication = false,
|
||||
Regions = ImmutableArray.Create("RU", "CIS"),
|
||||
DefaultPriority = 100,
|
||||
Tags = ImmutableArray.Create("fstec", "bdu", "russia", "cis")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition Nkcki = new()
|
||||
{
|
||||
Id = "nkcki",
|
||||
DisplayName = "NKCKI (Russia)",
|
||||
Category = SourceCategory.Cert,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "National Coordination Center for Computer Incidents",
|
||||
BaseEndpoint = "https://safe-surf.ru/",
|
||||
HealthCheckEndpoint = "https://safe-surf.ru/",
|
||||
HttpClientName = "NkckiClient",
|
||||
RequiresAuthentication = false,
|
||||
Regions = ImmutableArray.Create("RU", "CIS"),
|
||||
DefaultPriority = 101,
|
||||
Tags = ImmutableArray.Create("nkcki", "russia", "cis", "cert")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition KasperskyIcs = new()
|
||||
{
|
||||
Id = "kaspersky-ics",
|
||||
DisplayName = "Kaspersky ICS-CERT",
|
||||
Category = SourceCategory.Ics,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Kaspersky Industrial Control Systems CERT",
|
||||
BaseEndpoint = "https://ics-cert.kaspersky.com/",
|
||||
HealthCheckEndpoint = "https://ics-cert.kaspersky.com/",
|
||||
HttpClientName = "KasperskyIcsClient",
|
||||
RequiresAuthentication = false,
|
||||
Regions = ImmutableArray.Create("RU", "CIS", "GLOBAL"),
|
||||
DefaultPriority = 102,
|
||||
Tags = ImmutableArray.Create("kaspersky", "ics", "russia", "cis", "scada")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition AstraLinux = new()
|
||||
{
|
||||
Id = "astra",
|
||||
DisplayName = "Astra Linux Security",
|
||||
Category = SourceCategory.Distribution,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Astra Linux FSTEC-certified distribution security updates",
|
||||
BaseEndpoint = "https://wiki.astralinux.ru/",
|
||||
HealthCheckEndpoint = "https://wiki.astralinux.ru/",
|
||||
HttpClientName = "AstraLinuxClient",
|
||||
RequiresAuthentication = false,
|
||||
Regions = ImmutableArray.Create("RU", "CIS"),
|
||||
DefaultPriority = 48,
|
||||
Tags = ImmutableArray.Create("astra", "distro", "linux", "fstec", "russia")
|
||||
};
|
||||
|
||||
// ===== Threat Intelligence =====
|
||||
|
||||
public static readonly SourceDefinition MitreAttack = new()
|
||||
{
|
||||
Id = "mitre-attack",
|
||||
DisplayName = "MITRE ATT&CK",
|
||||
Category = SourceCategory.Threat,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "MITRE ATT&CK adversary tactics and techniques knowledge base",
|
||||
BaseEndpoint = "https://raw.githubusercontent.com/mitre/cti/master/",
|
||||
HealthCheckEndpoint = "https://raw.githubusercontent.com/mitre/cti/master/README.md",
|
||||
HttpClientName = "MitreAttackClient",
|
||||
RequiresAuthentication = false,
|
||||
DocumentationUrl = "https://attack.mitre.org/",
|
||||
DefaultPriority = 140,
|
||||
Tags = ImmutableArray.Create("mitre", "attack", "threat-intel", "tactics")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition MitreD3fend = new()
|
||||
{
|
||||
Id = "mitre-d3fend",
|
||||
DisplayName = "MITRE D3FEND",
|
||||
Category = SourceCategory.Threat,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "MITRE D3FEND defensive techniques knowledge base",
|
||||
BaseEndpoint = "https://d3fend.mitre.org/api/",
|
||||
HealthCheckEndpoint = "https://d3fend.mitre.org/api/",
|
||||
HttpClientName = "MitreD3fendClient",
|
||||
RequiresAuthentication = false,
|
||||
DocumentationUrl = "https://d3fend.mitre.org/",
|
||||
DefaultPriority = 142,
|
||||
Tags = ImmutableArray.Create("mitre", "d3fend", "threat-intel", "defensive")
|
||||
};
|
||||
|
||||
// ===== StellaOps Mirror =====
|
||||
|
||||
public static readonly SourceDefinition StellaMirror = new()
|
||||
@@ -924,14 +1376,32 @@ public static class SourceDefinitions
|
||||
Nvd, Osv, Ghsa, Cve, Epss, Kev,
|
||||
// Vendor advisories
|
||||
RedHat, Microsoft, Amazon, Google, Oracle, Apple, Cisco, Fortinet, Juniper, Palo, Vmware,
|
||||
// Cloud provider advisories
|
||||
Aws, Azure, Gcp,
|
||||
// Linux distributions
|
||||
Debian, Ubuntu, Alpine, Suse, Rhel, Centos, Fedora, Arch, Gentoo,
|
||||
Debian, Ubuntu, Alpine, Suse, Rhel, Centos, Fedora, Arch, Gentoo, AstraLinux,
|
||||
// Ecosystems
|
||||
Npm, PyPi, Go, RubyGems, Nuget, Maven, Crates, Packagist, Hex,
|
||||
// Package manager native
|
||||
RustSec, PyPa, GoVuln, BundlerAudit,
|
||||
// CSAF/VEX
|
||||
Csaf, CsafTc, Vex,
|
||||
// Exploit databases
|
||||
ExploitDb, PocGithub, Metasploit,
|
||||
// Container sources
|
||||
DockerOfficial, Chainguard,
|
||||
// Hardware/firmware
|
||||
Intel, Amd, Arm,
|
||||
// ICS/SCADA
|
||||
Siemens, KasperskyIcs,
|
||||
// CERTs
|
||||
CertFr, CertDe, CertAt, CertBe, CertCh, CertEu, JpCert, UsCert,
|
||||
// Additional CERTs
|
||||
CertUa, CertPl, AusCert, KrCert, CertIn,
|
||||
// Russian/CIS
|
||||
FstecBdu, Nkcki,
|
||||
// Threat intelligence
|
||||
MitreAttack, MitreD3fend,
|
||||
// Mirrors
|
||||
StellaMirror);
|
||||
|
||||
|
||||
@@ -156,6 +156,206 @@ public static class SourcesServiceCollectionExtensions
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// Exploit-DB client
|
||||
services.AddHttpClient("ExploitDbClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://gitlab.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// PoC-in-GitHub client
|
||||
services.AddHttpClient("PocGithubClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://api.github.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.DefaultRequestHeaders.Add("User-Agent", "StellaOps-Concelier");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// Metasploit client
|
||||
services.AddHttpClient("MetasploitClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://raw.githubusercontent.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// Cloud provider clients
|
||||
services.AddHttpClient("AwsClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://aws.amazon.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("AzureClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://api.msrc.microsoft.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("GcpClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://cloud.google.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// Container clients
|
||||
services.AddHttpClient("DockerOfficialClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://hub.docker.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("ChainguardClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://images.chainguard.dev/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// Hardware clients
|
||||
services.AddHttpClient("IntelClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://www.intel.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("AmdClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://www.amd.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("ArmClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://developer.arm.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("SiemensClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://cert-portal.siemens.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// Package manager native clients
|
||||
services.AddHttpClient("RustSecClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://raw.githubusercontent.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("PyPaClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://raw.githubusercontent.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("GoVulnClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://vuln.go.dev/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("BundlerAuditClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://raw.githubusercontent.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// Additional CERT clients
|
||||
services.AddHttpClient("CertUaClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://cert.gov.ua/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("CertPlClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://cert.pl/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("AusCertClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://auscert.org.au/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("KrCertClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://www.krcert.or.kr/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("CertInClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://www.cert-in.org.in/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// Russian/CIS clients
|
||||
services.AddHttpClient("FstecBduClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://bdu.fstec.ru/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("NkckiClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://safe-surf.ru/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("KasperskyIcsClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://ics-cert.kaspersky.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("AstraLinuxClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://wiki.astralinux.ru/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// Threat intelligence clients
|
||||
services.AddHttpClient("MitreAttackClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://raw.githubusercontent.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("MitreD3fendClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://d3fend.mitre.org/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// StellaOps Mirror client
|
||||
services.AddHttpClient("StellaMirrorClient", client =>
|
||||
{
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace StellaOps.Excititor.Core;
|
||||
|
||||
@@ -29,6 +31,16 @@ public sealed class MirrorDistributionOptions
|
||||
/// </summary>
|
||||
public string? TargetRepository { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Interval in minutes for periodic mirror bundle refresh. Default: 60.
|
||||
/// </summary>
|
||||
public int RefreshIntervalMinutes { get; set; } = 60;
|
||||
|
||||
/// <summary>
|
||||
/// Whether automatic bundle refresh is enabled. Disable for air-gap imports.
|
||||
/// </summary>
|
||||
public bool AutoRefreshEnabled { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Signing configuration applied to generated bundle payloads.
|
||||
/// </summary>
|
||||
@@ -76,8 +88,83 @@ public sealed class MirrorExportOptions
|
||||
public int? Offset { get; set; } = null;
|
||||
|
||||
public string? View { get; set; } = null;
|
||||
|
||||
/// <summary>
|
||||
/// Resolves filter values, expanding category/tag shorthands and comma-separated values
|
||||
/// into normalized multi-value lists. Source definitions are required for resolving
|
||||
/// <c>sourceCategory</c> and <c>sourceTag</c> shorthands; pass <c>null</c> when
|
||||
/// category/tag expansion is not needed.
|
||||
/// </summary>
|
||||
/// <param name="sourceDefinitions">
|
||||
/// Optional catalog of source definitions used to resolve <c>sourceCategory</c> and
|
||||
/// <c>sourceTag</c> filter values. When <c>null</c>, those filter keys are treated as
|
||||
/// plain comma-separated values.
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// A dictionary mapping each filter key to an ordered list of resolved values.
|
||||
/// Multi-value filters (comma-separated, category-expanded, tag-expanded) produce
|
||||
/// multiple entries. Values are sorted alphabetically for deterministic signatures.
|
||||
/// </returns>
|
||||
public Dictionary<string, IReadOnlyList<string>> ResolveFilters(
|
||||
IReadOnlyList<MirrorSourceDefinitionDescriptor>? sourceDefinitions = null)
|
||||
{
|
||||
var resolved = new Dictionary<string, IReadOnlyList<string>>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var (key, value) in Filters)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value)) continue;
|
||||
|
||||
if (key.Equals("sourceCategory", StringComparison.OrdinalIgnoreCase) && sourceDefinitions is not null)
|
||||
{
|
||||
// Resolve category to source IDs
|
||||
var matchingIds = sourceDefinitions
|
||||
.Where(s => s.Category.Equals(value, StringComparison.OrdinalIgnoreCase))
|
||||
.Select(s => s.Id)
|
||||
.OrderBy(id => id, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
if (matchingIds.Count > 0)
|
||||
{
|
||||
resolved["sourceVendor"] = matchingIds;
|
||||
}
|
||||
}
|
||||
else if (key.Equals("sourceTag", StringComparison.OrdinalIgnoreCase) && sourceDefinitions is not null)
|
||||
{
|
||||
// Resolve tag to source IDs
|
||||
var matchingIds = sourceDefinitions
|
||||
.Where(s => s.Tags.Contains(value, StringComparer.OrdinalIgnoreCase))
|
||||
.Select(s => s.Id)
|
||||
.OrderBy(id => id, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
if (matchingIds.Count > 0)
|
||||
{
|
||||
resolved["sourceVendor"] = matchingIds;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Split comma-separated values
|
||||
var values = value.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
|
||||
.OrderBy(v => v, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
resolved[key] = values;
|
||||
}
|
||||
}
|
||||
|
||||
return resolved;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Lightweight descriptor for source definitions used by <see cref="MirrorExportOptions.ResolveFilters"/>.
|
||||
/// Decouples Excititor.Core from Concelier.Core while allowing category/tag resolution.
|
||||
/// </summary>
|
||||
public sealed record MirrorSourceDefinitionDescriptor(
|
||||
string Id,
|
||||
string Category,
|
||||
IReadOnlyList<string> Tags);
|
||||
|
||||
public sealed class MirrorSigningOptions
|
||||
{
|
||||
/// <summary>
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace StellaOps.Excititor.Core;
|
||||
@@ -11,7 +12,32 @@ public sealed record MirrorExportPlan(
|
||||
|
||||
public static class MirrorExportPlanner
|
||||
{
|
||||
/// <summary>
|
||||
/// Builds an export plan from the given options, expanding multi-value and
|
||||
/// comma-separated filter values into individual <see cref="VexQueryFilter"/> entries.
|
||||
/// This overload does not resolve <c>sourceCategory</c>/<c>sourceTag</c> shorthands.
|
||||
/// </summary>
|
||||
public static bool TryBuild(MirrorExportOptions exportOptions, out MirrorExportPlan plan, out string? error)
|
||||
=> TryBuild(exportOptions, sourceDefinitions: null, out plan, out error);
|
||||
|
||||
/// <summary>
|
||||
/// Builds an export plan from the given options, expanding multi-value filters,
|
||||
/// comma-separated values, and optionally resolving <c>sourceCategory</c>/<c>sourceTag</c>
|
||||
/// shorthands when <paramref name="sourceDefinitions"/> is provided.
|
||||
/// </summary>
|
||||
/// <param name="exportOptions">The export options containing raw filter configuration.</param>
|
||||
/// <param name="sourceDefinitions">
|
||||
/// Optional source catalog for resolving category/tag shorthands. Pass <c>null</c> to
|
||||
/// skip category/tag resolution (comma-separated expansion still applies).
|
||||
/// </param>
|
||||
/// <param name="plan">The resulting export plan when successful.</param>
|
||||
/// <param name="error">An error code when the build fails.</param>
|
||||
/// <returns><c>true</c> if the plan was built successfully; otherwise <c>false</c>.</returns>
|
||||
public static bool TryBuild(
|
||||
MirrorExportOptions exportOptions,
|
||||
IReadOnlyList<MirrorSourceDefinitionDescriptor>? sourceDefinitions,
|
||||
out MirrorExportPlan plan,
|
||||
out string? error)
|
||||
{
|
||||
if (exportOptions is null)
|
||||
{
|
||||
@@ -35,7 +61,9 @@ public static class MirrorExportPlanner
|
||||
return false;
|
||||
}
|
||||
|
||||
var filters = exportOptions.Filters.Select(pair => new VexQueryFilter(pair.Key, pair.Value));
|
||||
// Resolve filters: expand comma-separated values and category/tag shorthands
|
||||
var resolvedFilters = exportOptions.ResolveFilters(sourceDefinitions);
|
||||
var filters = ExpandResolvedFilters(resolvedFilters);
|
||||
var sorts = exportOptions.Sort.Select(pair => new VexQuerySort(pair.Key, pair.Value));
|
||||
var query = VexQuery.Create(filters, sorts, exportOptions.Limit, exportOptions.Offset, exportOptions.View);
|
||||
var signature = VexQuerySignature.FromQuery(query);
|
||||
@@ -44,4 +72,23 @@ public static class MirrorExportPlanner
|
||||
error = null;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Expands resolved multi-value filters into individual <see cref="VexQueryFilter"/>
|
||||
/// entries. For a key with multiple values (e.g., <c>sourceVendor</c> = ["alpine","debian","ubuntu"]),
|
||||
/// each value produces a separate <see cref="VexQueryFilter"/> with the same key.
|
||||
/// This ensures OR-semantics: a source matches if its vendor ID equals ANY of the values.
|
||||
/// Values are emitted in sorted order for deterministic query signatures.
|
||||
/// </summary>
|
||||
internal static IEnumerable<VexQueryFilter> ExpandResolvedFilters(
|
||||
Dictionary<string, IReadOnlyList<string>> resolvedFilters)
|
||||
{
|
||||
foreach (var (key, values) in resolvedFilters.OrderBy(kvp => kvp.Key, StringComparer.Ordinal))
|
||||
{
|
||||
foreach (var value in values)
|
||||
{
|
||||
yield return new VexQueryFilter(key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,443 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
|
||||
namespace StellaOps.Excititor.Core;
|
||||
|
||||
/// <summary>
|
||||
/// Background service that periodically checks configured mirror domains for stale
|
||||
/// export bundles and triggers regeneration when source data has been updated since
|
||||
/// the last bundle generation. Designed for air-gap awareness: can be fully disabled
|
||||
/// via <see cref="MirrorDistributionOptions.AutoRefreshEnabled"/>.
|
||||
/// </summary>
|
||||
public sealed class MirrorExportScheduler : BackgroundService
|
||||
{
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly IOptionsMonitor<MirrorDistributionOptions> _optionsMonitor;
|
||||
private readonly ILogger<MirrorExportScheduler> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ConcurrentDictionary<string, DomainGenerationStatus> _domainStatus = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of <see cref="MirrorExportScheduler"/>.
|
||||
/// </summary>
|
||||
public MirrorExportScheduler(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
IOptionsMonitor<MirrorDistributionOptions> optionsMonitor,
|
||||
ILogger<MirrorExportScheduler> logger,
|
||||
TimeProvider timeProvider)
|
||||
{
|
||||
_scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory));
|
||||
_optionsMonitor = optionsMonitor ?? throw new ArgumentNullException(nameof(optionsMonitor));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a snapshot of generation status for all tracked domains.
|
||||
/// </summary>
|
||||
public IReadOnlyDictionary<string, DomainGenerationStatus> GetDomainStatuses()
|
||||
=> _domainStatus.ToDictionary(kvp => kvp.Key, kvp => kvp.Value, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
var options = _optionsMonitor.CurrentValue;
|
||||
|
||||
if (!options.AutoRefreshEnabled)
|
||||
{
|
||||
_logger.LogInformation("MirrorExportScheduler is disabled (AutoRefreshEnabled=false). Exiting.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!options.Enabled)
|
||||
{
|
||||
_logger.LogInformation("MirrorExportScheduler is disabled (Mirror distribution disabled). Exiting.");
|
||||
return;
|
||||
}
|
||||
|
||||
var intervalMinutes = Math.Max(options.RefreshIntervalMinutes, 1);
|
||||
|
||||
_logger.LogInformation(
|
||||
"MirrorExportScheduler started. Refresh interval: {IntervalMinutes} min, Domains: {DomainCount}",
|
||||
intervalMinutes,
|
||||
options.Domains.Count);
|
||||
|
||||
// Allow other services to initialize before the first sweep
|
||||
await Task.Delay(TimeSpan.FromSeconds(15), stoppingToken).ConfigureAwait(false);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var currentOptions = _optionsMonitor.CurrentValue;
|
||||
|
||||
if (!currentOptions.AutoRefreshEnabled)
|
||||
{
|
||||
_logger.LogInformation("MirrorExportScheduler auto-refresh disabled at runtime. Stopping refresh loop.");
|
||||
break;
|
||||
}
|
||||
|
||||
if (!currentOptions.Enabled)
|
||||
{
|
||||
_logger.LogDebug("Mirror distribution disabled; skipping refresh cycle.");
|
||||
}
|
||||
else
|
||||
{
|
||||
await RefreshAllDomainsAsync(currentOptions, stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
intervalMinutes = Math.Max(currentOptions.RefreshIntervalMinutes, 1);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Unhandled error in MirrorExportScheduler refresh cycle.");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMinutes(intervalMinutes), stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation("MirrorExportScheduler stopped.");
|
||||
}
|
||||
|
||||
private async Task RefreshAllDomainsAsync(MirrorDistributionOptions options, CancellationToken cancellationToken)
|
||||
{
|
||||
if (options.Domains.Count == 0)
|
||||
{
|
||||
_logger.LogDebug("No mirror domains configured; skipping refresh cycle.");
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogDebug("Starting mirror refresh cycle for {DomainCount} domain(s).", options.Domains.Count);
|
||||
|
||||
foreach (var domain in options.Domains)
|
||||
{
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await RefreshDomainAsync(options, domain, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Failed to refresh mirror domain {DomainId}. Continuing with remaining domains.",
|
||||
domain.Id);
|
||||
|
||||
UpdateDomainStatus(domain.Id, status => status with
|
||||
{
|
||||
LastError = ex.Message,
|
||||
LastErrorAt = _timeProvider.GetUtcNow(),
|
||||
ConsecutiveFailures = status.ConsecutiveFailures + 1,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogDebug("Mirror refresh cycle completed.");
|
||||
}
|
||||
|
||||
private async Task RefreshDomainAsync(
|
||||
MirrorDistributionOptions options,
|
||||
MirrorDomainOptions domain,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (domain.Exports.Count == 0)
|
||||
{
|
||||
_logger.LogDebug("Domain {DomainId} has no exports; skipping.", domain.Id);
|
||||
return;
|
||||
}
|
||||
|
||||
var stalePlanCount = 0;
|
||||
var regeneratedCount = 0;
|
||||
var domainStartTime = Stopwatch.GetTimestamp();
|
||||
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var connectorStateRepo = scope.ServiceProvider.GetService<IVexConnectorStateRepository>();
|
||||
|
||||
// Resolve the latest source update time from connector state
|
||||
DateTimeOffset? latestSourceUpdate = null;
|
||||
if (connectorStateRepo is not null)
|
||||
{
|
||||
var connectorStates = await connectorStateRepo.ListAsync(cancellationToken).ConfigureAwait(false);
|
||||
latestSourceUpdate = connectorStates
|
||||
.Where(s => s.LastUpdated.HasValue)
|
||||
.Max(s => s.LastUpdated);
|
||||
}
|
||||
|
||||
foreach (var exportOption in domain.Exports)
|
||||
{
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (!MirrorExportPlanner.TryBuild(exportOption, out var plan, out var error))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Skipping export {ExportKey} in domain {DomainId}: {Error}",
|
||||
exportOption.Key,
|
||||
domain.Id,
|
||||
error);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var isStale = await CheckExportStalenessAsync(
|
||||
scope.ServiceProvider,
|
||||
plan,
|
||||
latestSourceUpdate,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!isStale)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Export {ExportKey} in domain {DomainId} is fresh; skipping regeneration.",
|
||||
plan.Key,
|
||||
domain.Id);
|
||||
continue;
|
||||
}
|
||||
|
||||
stalePlanCount++;
|
||||
|
||||
_logger.LogInformation(
|
||||
"Export {ExportKey} in domain {DomainId} is stale. Triggering bundle regeneration.",
|
||||
plan.Key,
|
||||
domain.Id);
|
||||
|
||||
var exportStartTime = Stopwatch.GetTimestamp();
|
||||
|
||||
await RegenerateExportAsync(
|
||||
scope.ServiceProvider,
|
||||
options,
|
||||
domain,
|
||||
plan,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var exportDuration = Stopwatch.GetElapsedTime(exportStartTime);
|
||||
regeneratedCount++;
|
||||
|
||||
_logger.LogInformation(
|
||||
"Regenerated export {ExportKey} in domain {DomainId} in {DurationMs:F0}ms.",
|
||||
plan.Key,
|
||||
domain.Id,
|
||||
exportDuration.TotalMilliseconds);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Failed to regenerate export {ExportKey} in domain {DomainId}. Continuing with remaining exports.",
|
||||
plan.Key,
|
||||
domain.Id);
|
||||
}
|
||||
}
|
||||
|
||||
var domainDuration = Stopwatch.GetElapsedTime(domainStartTime);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
if (regeneratedCount > 0)
|
||||
{
|
||||
UpdateDomainStatus(domain.Id, status => status with
|
||||
{
|
||||
LastGeneratedAt = now,
|
||||
LastRefreshDurationMs = domainDuration.TotalMilliseconds,
|
||||
ExportsRegenerated = regeneratedCount,
|
||||
StaleExportsFound = stalePlanCount,
|
||||
ConsecutiveFailures = 0,
|
||||
LastError = null,
|
||||
LastErrorAt = null,
|
||||
});
|
||||
|
||||
_logger.LogInformation(
|
||||
"Domain {DomainId} refresh complete: {Regenerated}/{Stale} exports regenerated in {DurationMs:F0}ms.",
|
||||
domain.Id,
|
||||
regeneratedCount,
|
||||
stalePlanCount,
|
||||
domainDuration.TotalMilliseconds);
|
||||
}
|
||||
else
|
||||
{
|
||||
UpdateDomainStatus(domain.Id, status => status with
|
||||
{
|
||||
LastCheckedAt = now,
|
||||
StaleExportsFound = stalePlanCount,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> CheckExportStalenessAsync(
|
||||
IServiceProvider services,
|
||||
MirrorExportPlan plan,
|
||||
DateTimeOffset? latestSourceUpdate,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// Resolve IVexExportStore from the DI container at runtime to avoid
|
||||
// a compile-time dependency from Excititor.Core on Excititor.Export
|
||||
var exportStore = services.GetService<IMirrorExportManifestLookup>();
|
||||
|
||||
if (exportStore is null)
|
||||
{
|
||||
// If no export store is registered, treat as stale (first run or unconfigured)
|
||||
_logger.LogDebug("No IMirrorExportManifestLookup registered; treating export {ExportKey} as stale.", plan.Key);
|
||||
return true;
|
||||
}
|
||||
|
||||
var lastManifest = await exportStore.FindManifestAsync(plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (lastManifest is null)
|
||||
{
|
||||
// No previous export: definitely stale
|
||||
return true;
|
||||
}
|
||||
|
||||
// If we have source update information, compare against last export time
|
||||
if (latestSourceUpdate.HasValue && lastManifest.CreatedAt < latestSourceUpdate.Value)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Export {ExportKey} created at {ExportCreatedAt:O} is older than latest source update at {SourceUpdate:O}.",
|
||||
plan.Key,
|
||||
lastManifest.CreatedAt,
|
||||
latestSourceUpdate.Value);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static async Task RegenerateExportAsync(
|
||||
IServiceProvider services,
|
||||
MirrorDistributionOptions options,
|
||||
MirrorDomainOptions domain,
|
||||
MirrorExportPlan plan,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var regenerator = services.GetService<IMirrorBundleRegenerator>();
|
||||
|
||||
if (regenerator is null)
|
||||
{
|
||||
// No regenerator registered; bundle generation deferred to the export pipeline
|
||||
return;
|
||||
}
|
||||
|
||||
await regenerator.RegenerateAsync(domain.Id, plan.Key, plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private void UpdateDomainStatus(string domainId, Func<DomainGenerationStatus, DomainGenerationStatus> updater)
|
||||
{
|
||||
_domainStatus.AddOrUpdate(
|
||||
domainId,
|
||||
_ => updater(DomainGenerationStatus.Empty),
|
||||
(_, existing) => updater(existing));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Status record tracking the last generation state for a mirror domain.
|
||||
/// Exposed via <see cref="MirrorExportScheduler.GetDomainStatuses"/> for
|
||||
/// observability and staleness reporting.
|
||||
/// </summary>
|
||||
public sealed record DomainGenerationStatus
|
||||
{
|
||||
public static readonly DomainGenerationStatus Empty = new();
|
||||
|
||||
/// <summary>When the domain was last successfully regenerated.</summary>
|
||||
public DateTimeOffset? LastGeneratedAt { get; init; }
|
||||
|
||||
/// <summary>When the domain was last checked (even if nothing was stale).</summary>
|
||||
public DateTimeOffset? LastCheckedAt { get; init; }
|
||||
|
||||
/// <summary>Duration of the last refresh cycle in milliseconds.</summary>
|
||||
public double LastRefreshDurationMs { get; init; }
|
||||
|
||||
/// <summary>Number of stale exports found in the last cycle.</summary>
|
||||
public int StaleExportsFound { get; init; }
|
||||
|
||||
/// <summary>Number of exports regenerated in the last cycle.</summary>
|
||||
public int ExportsRegenerated { get; init; }
|
||||
|
||||
/// <summary>Number of consecutive domain-level failures.</summary>
|
||||
public int ConsecutiveFailures { get; init; }
|
||||
|
||||
/// <summary>Last error message, if any.</summary>
|
||||
public string? LastError { get; init; }
|
||||
|
||||
/// <summary>When the last error occurred.</summary>
|
||||
public DateTimeOffset? LastErrorAt { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Abstraction for looking up the latest export manifest by signature and format.
|
||||
/// This decouples <see cref="MirrorExportScheduler"/> (Excititor.Core) from the
|
||||
/// concrete <c>IVexExportStore</c> in Excititor.Export without introducing a
|
||||
/// compile-time dependency.
|
||||
/// </summary>
|
||||
public interface IMirrorExportManifestLookup
|
||||
{
|
||||
/// <summary>
|
||||
/// Finds the latest export manifest matching the given query signature and format.
|
||||
/// </summary>
|
||||
ValueTask<MirrorExportManifestSnapshot?> FindManifestAsync(
|
||||
VexQuerySignature signature,
|
||||
VexExportFormat format,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Lightweight snapshot of an export manifest used for staleness comparisons.
|
||||
/// </summary>
|
||||
public sealed record MirrorExportManifestSnapshot(
|
||||
string ExportId,
|
||||
DateTimeOffset CreatedAt,
|
||||
long SizeBytes,
|
||||
int ClaimCount);
|
||||
|
||||
/// <summary>
|
||||
/// Abstraction for triggering bundle regeneration from the scheduler.
|
||||
/// Implemented by the export pipeline (Excititor.Export or WebService layer)
|
||||
/// to perform the actual VEX query, serialization, and bundle publishing.
|
||||
/// </summary>
|
||||
public interface IMirrorBundleRegenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Regenerates the export bundle for the given domain, export key, query signature, and format.
|
||||
/// </summary>
|
||||
Task RegenerateAsync(
|
||||
string domainId,
|
||||
string exportKey,
|
||||
VexQuerySignature signature,
|
||||
VexExportFormat format,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -15,6 +15,7 @@
|
||||
<PackageReference Include="Microsoft.Extensions.Http" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
|
||||
@@ -49,6 +49,15 @@ public interface IVexSourceRepository
|
||||
string? errorMessage = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Updates failure tracking fields for exponential backoff.
|
||||
/// </summary>
|
||||
Task UpdateFailureTrackingAsync(
|
||||
string sourceId,
|
||||
int consecutiveFailures,
|
||||
DateTimeOffset? nextEligiblePollAt,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Deletes a source by its ID.
|
||||
/// </summary>
|
||||
|
||||
@@ -68,9 +68,28 @@ public sealed class VexIngestionScheduler : BackgroundService
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation("Found {Count} sources due for polling", dueSources.Count);
|
||||
var utcNow = DateTimeOffset.UtcNow;
|
||||
var eligibleSources = dueSources
|
||||
.Where(s => s.NextEligiblePollAt == null || s.NextEligiblePollAt <= utcNow)
|
||||
.ToList();
|
||||
|
||||
var tasks = dueSources.Select(source => PollSourceWithThrottlingAsync(source, cancellationToken));
|
||||
var skippedCount = dueSources.Count - eligibleSources.Count;
|
||||
if (skippedCount > 0)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Skipped {SkippedCount} sources due to backoff. {EligibleCount} sources eligible for polling",
|
||||
skippedCount, eligibleSources.Count);
|
||||
}
|
||||
|
||||
if (eligibleSources.Count == 0)
|
||||
{
|
||||
_logger.LogDebug("No eligible sources after backoff filtering");
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation("Found {Count} sources due for polling", eligibleSources.Count);
|
||||
|
||||
var tasks = eligibleSources.Select(source => PollSourceWithThrottlingAsync(source, cancellationToken));
|
||||
await Task.WhenAll(tasks);
|
||||
}
|
||||
|
||||
@@ -100,6 +119,13 @@ public sealed class VexIngestionScheduler : BackgroundService
|
||||
DateTimeOffset.UtcNow,
|
||||
null,
|
||||
cancellationToken);
|
||||
|
||||
// Reset backoff on success
|
||||
if (source.ConsecutiveFailures > 0)
|
||||
{
|
||||
await _sourceRepository.UpdateFailureTrackingAsync(
|
||||
source.SourceId, 0, null, cancellationToken);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -113,6 +139,23 @@ public sealed class VexIngestionScheduler : BackgroundService
|
||||
DateTimeOffset.UtcNow,
|
||||
result.ErrorMessage,
|
||||
cancellationToken);
|
||||
|
||||
var failures = source.ConsecutiveFailures + 1;
|
||||
var backoff = _options.SourceBackoff;
|
||||
var rawBackoff = backoff.InitialBackoffSeconds * Math.Pow(backoff.BackoffMultiplier, failures - 1);
|
||||
var capped = Math.Min(rawBackoff, backoff.MaxBackoffSeconds);
|
||||
var jittered = capped + capped * backoff.JitterFactor * (Random.Shared.NextDouble() * 2 - 1);
|
||||
var nextEligible = DateTimeOffset.UtcNow.AddSeconds(jittered);
|
||||
|
||||
if (failures >= backoff.MaxConsecutiveFailures)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Source {SourceId} has failed {Failures} consecutive times. Next retry at {NextEligible}",
|
||||
source.SourceId, failures, nextEligible);
|
||||
}
|
||||
|
||||
await _sourceRepository.UpdateFailureTrackingAsync(
|
||||
source.SourceId, failures, nextEligible, cancellationToken);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
@@ -124,6 +167,23 @@ public sealed class VexIngestionScheduler : BackgroundService
|
||||
DateTimeOffset.UtcNow,
|
||||
ex.Message,
|
||||
cancellationToken);
|
||||
|
||||
var failures = source.ConsecutiveFailures + 1;
|
||||
var backoff = _options.SourceBackoff;
|
||||
var rawBackoff = backoff.InitialBackoffSeconds * Math.Pow(backoff.BackoffMultiplier, failures - 1);
|
||||
var capped = Math.Min(rawBackoff, backoff.MaxBackoffSeconds);
|
||||
var jittered = capped + capped * backoff.JitterFactor * (Random.Shared.NextDouble() * 2 - 1);
|
||||
var nextEligible = DateTimeOffset.UtcNow.AddSeconds(jittered);
|
||||
|
||||
if (failures >= backoff.MaxConsecutiveFailures)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Source {SourceId} has failed {Failures} consecutive times. Next retry at {NextEligible}",
|
||||
source.SourceId, failures, nextEligible);
|
||||
}
|
||||
|
||||
await _sourceRepository.UpdateFailureTrackingAsync(
|
||||
source.SourceId, failures, nextEligible, cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
|
||||
@@ -68,6 +68,12 @@ public sealed record VexSource
|
||||
/// When the source configuration was last updated.
|
||||
/// </summary>
|
||||
public DateTimeOffset? UpdatedAt { get; init; }
|
||||
|
||||
/// <summary>Number of consecutive polling failures.</summary>
|
||||
public int ConsecutiveFailures { get; init; }
|
||||
|
||||
/// <summary>Next eligible poll time after backoff. Null means eligible immediately.</summary>
|
||||
public DateTimeOffset? NextEligiblePollAt { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -71,6 +71,32 @@ public sealed class VexHubOptions
|
||||
/// Configuration for distribution/export behavior.
|
||||
/// </summary>
|
||||
public DistributionOptions Distribution { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for source failure backoff behavior.
|
||||
/// </summary>
|
||||
public SourceBackoffOptions SourceBackoff { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for source failure backoff behavior.
|
||||
/// </summary>
|
||||
public sealed class SourceBackoffOptions
|
||||
{
|
||||
/// <summary>Initial backoff in seconds after first failure.</summary>
|
||||
public int InitialBackoffSeconds { get; set; } = 3600;
|
||||
|
||||
/// <summary>Maximum backoff in seconds.</summary>
|
||||
public int MaxBackoffSeconds { get; set; } = 86400;
|
||||
|
||||
/// <summary>Multiplier for each consecutive failure.</summary>
|
||||
public double BackoffMultiplier { get; set; } = 2.0;
|
||||
|
||||
/// <summary>Jitter factor to avoid thundering herd.</summary>
|
||||
public double JitterFactor { get; set; } = 0.1;
|
||||
|
||||
/// <summary>Max failures before logging a warning (source stays backed off, not disabled).</summary>
|
||||
public int MaxConsecutiveFailures { get; set; } = 10;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
-- Migration: 002_add_source_backoff_columns
|
||||
-- Sprint: Advisory & VEX Source Management
|
||||
-- Adds failure tracking columns for exponential backoff
|
||||
|
||||
ALTER TABLE vexhub.vex_sources
|
||||
ADD COLUMN IF NOT EXISTS consecutive_failures INT NOT NULL DEFAULT 0,
|
||||
ADD COLUMN IF NOT EXISTS next_eligible_poll_at TIMESTAMPTZ NULL;
|
||||
Reference in New Issue
Block a user