Expand advisory source catalog to 75 sources and add mirror management backend
Source catalog: add 28 sources across 6 new categories (Exploit, Container, Hardware, ICS, PackageManager, additional CERTs) plus RU/CIS promotion and threat intel frameworks. Register 25 new HTTP clients. Source management API: 9 endpoints under /api/v1/sources for catalog browsing, connectivity checks, and enable/disable controls. Mirror domain API: 12 endpoints under /api/v1/mirror for domain CRUD, export management, on-demand bundle generation, and connectivity testing. Filter model: multi-value sourceVendor (comma-separated OR), sourceCategory and sourceTag shorthand resolution via ResolveFilters(). Backward-compatible with existing single-value filters. Deterministic query signatures. Mirror export scheduler: BackgroundService with configurable refresh interval, per-domain staleness detection, error isolation, and air-gap disable toggle. VEX ingestion backoff: exponential backoff for failed sources (1hr → 24hr cap) with jitter. New DB migration for tracking columns. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
master
@@ -128,6 +128,21 @@ public enum SourceCategory
|
||||
/// <summary>Exploit and threat intelligence sources.</summary>
|
||||
Threat,
|
||||
|
||||
/// <summary>Exploit databases and PoC sources.</summary>
|
||||
Exploit,
|
||||
|
||||
/// <summary>Container image advisory sources.</summary>
|
||||
Container,
|
||||
|
||||
/// <summary>Hardware and firmware PSIRT advisories.</summary>
|
||||
Hardware,
|
||||
|
||||
/// <summary>Industrial control systems and SCADA advisories.</summary>
|
||||
Ics,
|
||||
|
||||
/// <summary>Package manager native advisory databases (cargo-audit, pip-audit, govulncheck, bundler-audit).</summary>
|
||||
PackageManager,
|
||||
|
||||
/// <summary>StellaOps mirrors.</summary>
|
||||
Mirror,
|
||||
|
||||
@@ -150,7 +165,10 @@ public enum SourceType
|
||||
LocalFile,
|
||||
|
||||
/// <summary>Custom/user-defined source.</summary>
|
||||
Custom
|
||||
Custom,
|
||||
|
||||
/// <summary>STIX/TAXII protocol feed.</summary>
|
||||
StixTaxii
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -894,6 +912,440 @@ public static class SourceDefinitions
|
||||
Tags = ImmutableArray.Create("cert", "us", "cisa")
|
||||
};
|
||||
|
||||
// ===== Exploit Databases =====
|
||||
|
||||
public static readonly SourceDefinition ExploitDb = new()
|
||||
{
|
||||
Id = "exploitdb",
|
||||
DisplayName = "Exploit-DB",
|
||||
Category = SourceCategory.Exploit,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Offensive Security Exploit Database",
|
||||
BaseEndpoint = "https://gitlab.com/exploit-database/exploitdb/-/raw/main/",
|
||||
HealthCheckEndpoint = "https://gitlab.com/exploit-database/exploitdb",
|
||||
HttpClientName = "ExploitDbClient",
|
||||
RequiresAuthentication = false,
|
||||
DocumentationUrl = "https://www.exploit-db.com/",
|
||||
DefaultPriority = 110,
|
||||
Tags = ImmutableArray.Create("exploit", "poc", "offensive")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition PocGithub = new()
|
||||
{
|
||||
Id = "poc-github",
|
||||
DisplayName = "PoC-in-GitHub",
|
||||
Category = SourceCategory.Exploit,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "GitHub repositories containing vulnerability PoCs",
|
||||
BaseEndpoint = "https://api.github.com/search/repositories",
|
||||
HealthCheckEndpoint = "https://api.github.com/zen",
|
||||
HttpClientName = "PocGithubClient",
|
||||
RequiresAuthentication = true,
|
||||
CredentialEnvVar = "GITHUB_PAT",
|
||||
DefaultPriority = 112,
|
||||
Tags = ImmutableArray.Create("exploit", "poc", "github")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition Metasploit = new()
|
||||
{
|
||||
Id = "metasploit",
|
||||
DisplayName = "Metasploit Modules",
|
||||
Category = SourceCategory.Exploit,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Rapid7 Metasploit Framework vulnerability modules",
|
||||
BaseEndpoint = "https://raw.githubusercontent.com/rapid7/metasploit-framework/master/",
|
||||
HealthCheckEndpoint = "https://raw.githubusercontent.com/rapid7/metasploit-framework/master/README.md",
|
||||
HttpClientName = "MetasploitClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 114,
|
||||
Tags = ImmutableArray.Create("exploit", "metasploit", "rapid7")
|
||||
};
|
||||
|
||||
// ===== Cloud Provider Advisories =====
|
||||
|
||||
public static readonly SourceDefinition Aws = new()
|
||||
{
|
||||
Id = "aws",
|
||||
DisplayName = "AWS Security Bulletins",
|
||||
Category = SourceCategory.Vendor,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Amazon Web Services security bulletins",
|
||||
BaseEndpoint = "https://aws.amazon.com/security/security-bulletins/",
|
||||
HealthCheckEndpoint = "https://aws.amazon.com/security/security-bulletins/",
|
||||
HttpClientName = "AwsClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 81,
|
||||
Tags = ImmutableArray.Create("aws", "vendor", "cloud")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition Azure = new()
|
||||
{
|
||||
Id = "azure",
|
||||
DisplayName = "Azure Security Advisories",
|
||||
Category = SourceCategory.Vendor,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Microsoft Azure security advisories",
|
||||
BaseEndpoint = "https://api.msrc.microsoft.com/sug/v2.0/en-US/",
|
||||
HealthCheckEndpoint = "https://api.msrc.microsoft.com/sug/v2.0/en-US/affectedProduct",
|
||||
HttpClientName = "AzureClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 82,
|
||||
Tags = ImmutableArray.Create("azure", "vendor", "cloud", "microsoft")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition Gcp = new()
|
||||
{
|
||||
Id = "gcp",
|
||||
DisplayName = "GCP Security Bulletins",
|
||||
Category = SourceCategory.Vendor,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Google Cloud Platform security bulletins",
|
||||
BaseEndpoint = "https://cloud.google.com/support/bulletins/",
|
||||
HealthCheckEndpoint = "https://cloud.google.com/support/bulletins/",
|
||||
HttpClientName = "GcpClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 83,
|
||||
Tags = ImmutableArray.Create("gcp", "vendor", "cloud", "google")
|
||||
};
|
||||
|
||||
// ===== Container Sources =====
|
||||
|
||||
public static readonly SourceDefinition DockerOfficial = new()
|
||||
{
|
||||
Id = "docker-official",
|
||||
DisplayName = "Docker Official CVEs",
|
||||
Category = SourceCategory.Container,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Docker Official Images CVE notices",
|
||||
BaseEndpoint = "https://hub.docker.com/v2/",
|
||||
HealthCheckEndpoint = "https://hub.docker.com/v2/",
|
||||
HttpClientName = "DockerOfficialClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 120,
|
||||
Tags = ImmutableArray.Create("docker", "container", "oci")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition Chainguard = new()
|
||||
{
|
||||
Id = "chainguard",
|
||||
DisplayName = "Chainguard Advisories",
|
||||
Category = SourceCategory.Container,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Chainguard hardened image advisories",
|
||||
BaseEndpoint = "https://images.chainguard.dev/",
|
||||
HealthCheckEndpoint = "https://images.chainguard.dev/",
|
||||
HttpClientName = "ChainguardClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 122,
|
||||
Tags = ImmutableArray.Create("chainguard", "container", "hardened")
|
||||
};
|
||||
|
||||
// ===== Hardware/Firmware =====
|
||||
|
||||
public static readonly SourceDefinition Intel = new()
|
||||
{
|
||||
Id = "intel",
|
||||
DisplayName = "Intel PSIRT",
|
||||
Category = SourceCategory.Hardware,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Intel Product Security Incident Response Team",
|
||||
BaseEndpoint = "https://www.intel.com/content/www/us/en/security-center/default.html",
|
||||
HealthCheckEndpoint = "https://www.intel.com/content/www/us/en/security-center/default.html",
|
||||
HttpClientName = "IntelClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 130,
|
||||
Tags = ImmutableArray.Create("intel", "hardware", "firmware", "cpu")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition Amd = new()
|
||||
{
|
||||
Id = "amd",
|
||||
DisplayName = "AMD Security",
|
||||
Category = SourceCategory.Hardware,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "AMD Product Security advisories",
|
||||
BaseEndpoint = "https://www.amd.com/en/resources/product-security.html",
|
||||
HealthCheckEndpoint = "https://www.amd.com/en/resources/product-security.html",
|
||||
HttpClientName = "AmdClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 132,
|
||||
Tags = ImmutableArray.Create("amd", "hardware", "firmware", "cpu")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition Arm = new()
|
||||
{
|
||||
Id = "arm",
|
||||
DisplayName = "ARM Security Center",
|
||||
Category = SourceCategory.Hardware,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "ARM Security Center advisories",
|
||||
BaseEndpoint = "https://developer.arm.com/Arm%20Security%20Center/",
|
||||
HealthCheckEndpoint = "https://developer.arm.com/Arm%20Security%20Center/",
|
||||
HttpClientName = "ArmClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 134,
|
||||
Tags = ImmutableArray.Create("arm", "hardware", "firmware", "cpu")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition Siemens = new()
|
||||
{
|
||||
Id = "siemens",
|
||||
DisplayName = "Siemens ProductCERT",
|
||||
Category = SourceCategory.Ics,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Siemens Product CERT ICS advisories",
|
||||
BaseEndpoint = "https://cert-portal.siemens.com/productcert/csaf/",
|
||||
HealthCheckEndpoint = "https://cert-portal.siemens.com/productcert/",
|
||||
HttpClientName = "SiemensClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 136,
|
||||
Tags = ImmutableArray.Create("siemens", "ics", "scada", "hardware")
|
||||
};
|
||||
|
||||
// ===== Package Manager Native Advisories =====
|
||||
|
||||
public static readonly SourceDefinition RustSec = new()
|
||||
{
|
||||
Id = "rustsec",
|
||||
DisplayName = "RustSec Advisory DB",
|
||||
Category = SourceCategory.PackageManager,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Rust Security Advisory Database (cargo-audit)",
|
||||
BaseEndpoint = "https://raw.githubusercontent.com/rustsec/advisory-db/main/",
|
||||
HealthCheckEndpoint = "https://raw.githubusercontent.com/rustsec/advisory-db/main/README.md",
|
||||
HttpClientName = "RustSecClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 63,
|
||||
Tags = ImmutableArray.Create("rustsec", "package-manager", "rust", "cargo")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition PyPa = new()
|
||||
{
|
||||
Id = "pypa",
|
||||
DisplayName = "PyPA Advisory DB",
|
||||
Category = SourceCategory.PackageManager,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Python Packaging Authority Advisory Database (pip-audit)",
|
||||
BaseEndpoint = "https://raw.githubusercontent.com/pypa/advisory-database/main/",
|
||||
HealthCheckEndpoint = "https://raw.githubusercontent.com/pypa/advisory-database/main/README.md",
|
||||
HttpClientName = "PyPaClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 53,
|
||||
Tags = ImmutableArray.Create("pypa", "package-manager", "python", "pip")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition GoVuln = new()
|
||||
{
|
||||
Id = "govuln",
|
||||
DisplayName = "Go Vuln DB",
|
||||
Category = SourceCategory.PackageManager,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Go Vulnerability Database (govulncheck)",
|
||||
BaseEndpoint = "https://vuln.go.dev/",
|
||||
HealthCheckEndpoint = "https://vuln.go.dev/",
|
||||
HttpClientName = "GoVulnClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 55,
|
||||
Tags = ImmutableArray.Create("govuln", "package-manager", "go", "golang")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition BundlerAudit = new()
|
||||
{
|
||||
Id = "bundler-audit",
|
||||
DisplayName = "Ruby Advisory DB",
|
||||
Category = SourceCategory.PackageManager,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Ruby Advisory Database (bundler-audit)",
|
||||
BaseEndpoint = "https://raw.githubusercontent.com/rubysec/ruby-advisory-db/main/",
|
||||
HealthCheckEndpoint = "https://raw.githubusercontent.com/rubysec/ruby-advisory-db/main/README.md",
|
||||
HttpClientName = "BundlerAuditClient",
|
||||
RequiresAuthentication = false,
|
||||
DefaultPriority = 57,
|
||||
Tags = ImmutableArray.Create("bundler", "package-manager", "ruby", "rubysec")
|
||||
};
|
||||
|
||||
// ===== Additional CERTs =====
|
||||
|
||||
public static readonly SourceDefinition CertUa = new()
|
||||
{
|
||||
Id = "cert-ua",
|
||||
DisplayName = "CERT-UA (Ukraine)",
|
||||
Category = SourceCategory.Cert,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Ukrainian Computer Emergency Response Team",
|
||||
BaseEndpoint = "https://cert.gov.ua/",
|
||||
HealthCheckEndpoint = "https://cert.gov.ua/",
|
||||
HttpClientName = "CertUaClient",
|
||||
RequiresAuthentication = false,
|
||||
Regions = ImmutableArray.Create("UA"),
|
||||
DefaultPriority = 95,
|
||||
Tags = ImmutableArray.Create("cert", "ukraine")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition CertPl = new()
|
||||
{
|
||||
Id = "cert-pl",
|
||||
DisplayName = "CERT.PL (Poland)",
|
||||
Category = SourceCategory.Cert,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Polish CERT",
|
||||
BaseEndpoint = "https://cert.pl/en/",
|
||||
HealthCheckEndpoint = "https://cert.pl/en/",
|
||||
HttpClientName = "CertPlClient",
|
||||
RequiresAuthentication = false,
|
||||
Regions = ImmutableArray.Create("PL", "EU"),
|
||||
DefaultPriority = 96,
|
||||
Tags = ImmutableArray.Create("cert", "poland", "eu")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition AusCert = new()
|
||||
{
|
||||
Id = "auscert",
|
||||
DisplayName = "AusCERT (Australia)",
|
||||
Category = SourceCategory.Cert,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Australian Cyber Security Centre CERT",
|
||||
BaseEndpoint = "https://auscert.org.au/",
|
||||
HealthCheckEndpoint = "https://auscert.org.au/",
|
||||
HttpClientName = "AusCertClient",
|
||||
RequiresAuthentication = false,
|
||||
Regions = ImmutableArray.Create("AU", "APAC"),
|
||||
DefaultPriority = 97,
|
||||
Tags = ImmutableArray.Create("cert", "australia", "apac")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition KrCert = new()
|
||||
{
|
||||
Id = "krcert",
|
||||
DisplayName = "KrCERT/CC (South Korea)",
|
||||
Category = SourceCategory.Cert,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Korean Computer Emergency Response Team",
|
||||
BaseEndpoint = "https://www.krcert.or.kr/",
|
||||
HealthCheckEndpoint = "https://www.krcert.or.kr/",
|
||||
HttpClientName = "KrCertClient",
|
||||
RequiresAuthentication = false,
|
||||
Regions = ImmutableArray.Create("KR", "APAC"),
|
||||
DefaultPriority = 98,
|
||||
Tags = ImmutableArray.Create("cert", "korea", "apac")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition CertIn = new()
|
||||
{
|
||||
Id = "cert-in",
|
||||
DisplayName = "CERT-In (India)",
|
||||
Category = SourceCategory.Cert,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Indian Computer Emergency Response Team",
|
||||
BaseEndpoint = "https://www.cert-in.org.in/",
|
||||
HealthCheckEndpoint = "https://www.cert-in.org.in/",
|
||||
HttpClientName = "CertInClient",
|
||||
RequiresAuthentication = false,
|
||||
Regions = ImmutableArray.Create("IN", "APAC"),
|
||||
DefaultPriority = 99,
|
||||
Tags = ImmutableArray.Create("cert", "india", "apac")
|
||||
};
|
||||
|
||||
// ===== Russian/CIS Sources =====
|
||||
|
||||
public static readonly SourceDefinition FstecBdu = new()
|
||||
{
|
||||
Id = "fstec-bdu",
|
||||
DisplayName = "FSTEC BDU (Russia)",
|
||||
Category = SourceCategory.Cert,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Federal Service for Technical and Export Control — Bank of Security Threats",
|
||||
BaseEndpoint = "https://bdu.fstec.ru/",
|
||||
HealthCheckEndpoint = "https://bdu.fstec.ru/",
|
||||
HttpClientName = "FstecBduClient",
|
||||
RequiresAuthentication = false,
|
||||
Regions = ImmutableArray.Create("RU", "CIS"),
|
||||
DefaultPriority = 100,
|
||||
Tags = ImmutableArray.Create("fstec", "bdu", "russia", "cis")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition Nkcki = new()
|
||||
{
|
||||
Id = "nkcki",
|
||||
DisplayName = "NKCKI (Russia)",
|
||||
Category = SourceCategory.Cert,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "National Coordination Center for Computer Incidents",
|
||||
BaseEndpoint = "https://safe-surf.ru/",
|
||||
HealthCheckEndpoint = "https://safe-surf.ru/",
|
||||
HttpClientName = "NkckiClient",
|
||||
RequiresAuthentication = false,
|
||||
Regions = ImmutableArray.Create("RU", "CIS"),
|
||||
DefaultPriority = 101,
|
||||
Tags = ImmutableArray.Create("nkcki", "russia", "cis", "cert")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition KasperskyIcs = new()
|
||||
{
|
||||
Id = "kaspersky-ics",
|
||||
DisplayName = "Kaspersky ICS-CERT",
|
||||
Category = SourceCategory.Ics,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Kaspersky Industrial Control Systems CERT",
|
||||
BaseEndpoint = "https://ics-cert.kaspersky.com/",
|
||||
HealthCheckEndpoint = "https://ics-cert.kaspersky.com/",
|
||||
HttpClientName = "KasperskyIcsClient",
|
||||
RequiresAuthentication = false,
|
||||
Regions = ImmutableArray.Create("RU", "CIS", "GLOBAL"),
|
||||
DefaultPriority = 102,
|
||||
Tags = ImmutableArray.Create("kaspersky", "ics", "russia", "cis", "scada")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition AstraLinux = new()
|
||||
{
|
||||
Id = "astra",
|
||||
DisplayName = "Astra Linux Security",
|
||||
Category = SourceCategory.Distribution,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "Astra Linux FSTEC-certified distribution security updates",
|
||||
BaseEndpoint = "https://wiki.astralinux.ru/",
|
||||
HealthCheckEndpoint = "https://wiki.astralinux.ru/",
|
||||
HttpClientName = "AstraLinuxClient",
|
||||
RequiresAuthentication = false,
|
||||
Regions = ImmutableArray.Create("RU", "CIS"),
|
||||
DefaultPriority = 48,
|
||||
Tags = ImmutableArray.Create("astra", "distro", "linux", "fstec", "russia")
|
||||
};
|
||||
|
||||
// ===== Threat Intelligence =====
|
||||
|
||||
public static readonly SourceDefinition MitreAttack = new()
|
||||
{
|
||||
Id = "mitre-attack",
|
||||
DisplayName = "MITRE ATT&CK",
|
||||
Category = SourceCategory.Threat,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "MITRE ATT&CK adversary tactics and techniques knowledge base",
|
||||
BaseEndpoint = "https://raw.githubusercontent.com/mitre/cti/master/",
|
||||
HealthCheckEndpoint = "https://raw.githubusercontent.com/mitre/cti/master/README.md",
|
||||
HttpClientName = "MitreAttackClient",
|
||||
RequiresAuthentication = false,
|
||||
DocumentationUrl = "https://attack.mitre.org/",
|
||||
DefaultPriority = 140,
|
||||
Tags = ImmutableArray.Create("mitre", "attack", "threat-intel", "tactics")
|
||||
};
|
||||
|
||||
public static readonly SourceDefinition MitreD3fend = new()
|
||||
{
|
||||
Id = "mitre-d3fend",
|
||||
DisplayName = "MITRE D3FEND",
|
||||
Category = SourceCategory.Threat,
|
||||
Type = SourceType.Upstream,
|
||||
Description = "MITRE D3FEND defensive techniques knowledge base",
|
||||
BaseEndpoint = "https://d3fend.mitre.org/api/",
|
||||
HealthCheckEndpoint = "https://d3fend.mitre.org/api/",
|
||||
HttpClientName = "MitreD3fendClient",
|
||||
RequiresAuthentication = false,
|
||||
DocumentationUrl = "https://d3fend.mitre.org/",
|
||||
DefaultPriority = 142,
|
||||
Tags = ImmutableArray.Create("mitre", "d3fend", "threat-intel", "defensive")
|
||||
};
|
||||
|
||||
// ===== StellaOps Mirror =====
|
||||
|
||||
public static readonly SourceDefinition StellaMirror = new()
|
||||
@@ -924,14 +1376,32 @@ public static class SourceDefinitions
|
||||
Nvd, Osv, Ghsa, Cve, Epss, Kev,
|
||||
// Vendor advisories
|
||||
RedHat, Microsoft, Amazon, Google, Oracle, Apple, Cisco, Fortinet, Juniper, Palo, Vmware,
|
||||
// Cloud provider advisories
|
||||
Aws, Azure, Gcp,
|
||||
// Linux distributions
|
||||
Debian, Ubuntu, Alpine, Suse, Rhel, Centos, Fedora, Arch, Gentoo,
|
||||
Debian, Ubuntu, Alpine, Suse, Rhel, Centos, Fedora, Arch, Gentoo, AstraLinux,
|
||||
// Ecosystems
|
||||
Npm, PyPi, Go, RubyGems, Nuget, Maven, Crates, Packagist, Hex,
|
||||
// Package manager native
|
||||
RustSec, PyPa, GoVuln, BundlerAudit,
|
||||
// CSAF/VEX
|
||||
Csaf, CsafTc, Vex,
|
||||
// Exploit databases
|
||||
ExploitDb, PocGithub, Metasploit,
|
||||
// Container sources
|
||||
DockerOfficial, Chainguard,
|
||||
// Hardware/firmware
|
||||
Intel, Amd, Arm,
|
||||
// ICS/SCADA
|
||||
Siemens, KasperskyIcs,
|
||||
// CERTs
|
||||
CertFr, CertDe, CertAt, CertBe, CertCh, CertEu, JpCert, UsCert,
|
||||
// Additional CERTs
|
||||
CertUa, CertPl, AusCert, KrCert, CertIn,
|
||||
// Russian/CIS
|
||||
FstecBdu, Nkcki,
|
||||
// Threat intelligence
|
||||
MitreAttack, MitreD3fend,
|
||||
// Mirrors
|
||||
StellaMirror);
|
||||
|
||||
|
||||
@@ -156,6 +156,206 @@ public static class SourcesServiceCollectionExtensions
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// Exploit-DB client
|
||||
services.AddHttpClient("ExploitDbClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://gitlab.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// PoC-in-GitHub client
|
||||
services.AddHttpClient("PocGithubClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://api.github.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.DefaultRequestHeaders.Add("User-Agent", "StellaOps-Concelier");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// Metasploit client
|
||||
services.AddHttpClient("MetasploitClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://raw.githubusercontent.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// Cloud provider clients
|
||||
services.AddHttpClient("AwsClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://aws.amazon.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("AzureClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://api.msrc.microsoft.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("GcpClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://cloud.google.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// Container clients
|
||||
services.AddHttpClient("DockerOfficialClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://hub.docker.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("ChainguardClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://images.chainguard.dev/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// Hardware clients
|
||||
services.AddHttpClient("IntelClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://www.intel.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("AmdClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://www.amd.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("ArmClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://developer.arm.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("SiemensClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://cert-portal.siemens.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// Package manager native clients
|
||||
services.AddHttpClient("RustSecClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://raw.githubusercontent.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("PyPaClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://raw.githubusercontent.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("GoVulnClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://vuln.go.dev/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("BundlerAuditClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://raw.githubusercontent.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// Additional CERT clients
|
||||
services.AddHttpClient("CertUaClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://cert.gov.ua/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("CertPlClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://cert.pl/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("AusCertClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://auscert.org.au/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("KrCertClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://www.krcert.or.kr/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("CertInClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://www.cert-in.org.in/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// Russian/CIS clients
|
||||
services.AddHttpClient("FstecBduClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://bdu.fstec.ru/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("NkckiClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://safe-surf.ru/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("KasperskyIcsClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://ics-cert.kaspersky.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("AstraLinuxClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://wiki.astralinux.ru/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// Threat intelligence clients
|
||||
services.AddHttpClient("MitreAttackClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://raw.githubusercontent.com/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
services.AddHttpClient("MitreD3fendClient", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://d3fend.mitre.org/");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
// StellaOps Mirror client
|
||||
services.AddHttpClient("StellaMirrorClient", client =>
|
||||
{
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace StellaOps.Excititor.Core;
|
||||
|
||||
@@ -29,6 +31,16 @@ public sealed class MirrorDistributionOptions
|
||||
/// </summary>
|
||||
public string? TargetRepository { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Interval in minutes for periodic mirror bundle refresh. Default: 60.
|
||||
/// </summary>
|
||||
public int RefreshIntervalMinutes { get; set; } = 60;
|
||||
|
||||
/// <summary>
|
||||
/// Whether automatic bundle refresh is enabled. Disable for air-gap imports.
|
||||
/// </summary>
|
||||
public bool AutoRefreshEnabled { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Signing configuration applied to generated bundle payloads.
|
||||
/// </summary>
|
||||
@@ -76,8 +88,83 @@ public sealed class MirrorExportOptions
|
||||
public int? Offset { get; set; } = null;
|
||||
|
||||
public string? View { get; set; } = null;
|
||||
|
||||
/// <summary>
|
||||
/// Resolves filter values, expanding category/tag shorthands and comma-separated values
|
||||
/// into normalized multi-value lists. Source definitions are required for resolving
|
||||
/// <c>sourceCategory</c> and <c>sourceTag</c> shorthands; pass <c>null</c> when
|
||||
/// category/tag expansion is not needed.
|
||||
/// </summary>
|
||||
/// <param name="sourceDefinitions">
|
||||
/// Optional catalog of source definitions used to resolve <c>sourceCategory</c> and
|
||||
/// <c>sourceTag</c> filter values. When <c>null</c>, those filter keys are treated as
|
||||
/// plain comma-separated values.
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// A dictionary mapping each filter key to an ordered list of resolved values.
|
||||
/// Multi-value filters (comma-separated, category-expanded, tag-expanded) produce
|
||||
/// multiple entries. Values are sorted alphabetically for deterministic signatures.
|
||||
/// </returns>
|
||||
public Dictionary<string, IReadOnlyList<string>> ResolveFilters(
|
||||
IReadOnlyList<MirrorSourceDefinitionDescriptor>? sourceDefinitions = null)
|
||||
{
|
||||
var resolved = new Dictionary<string, IReadOnlyList<string>>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var (key, value) in Filters)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value)) continue;
|
||||
|
||||
if (key.Equals("sourceCategory", StringComparison.OrdinalIgnoreCase) && sourceDefinitions is not null)
|
||||
{
|
||||
// Resolve category to source IDs
|
||||
var matchingIds = sourceDefinitions
|
||||
.Where(s => s.Category.Equals(value, StringComparison.OrdinalIgnoreCase))
|
||||
.Select(s => s.Id)
|
||||
.OrderBy(id => id, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
if (matchingIds.Count > 0)
|
||||
{
|
||||
resolved["sourceVendor"] = matchingIds;
|
||||
}
|
||||
}
|
||||
else if (key.Equals("sourceTag", StringComparison.OrdinalIgnoreCase) && sourceDefinitions is not null)
|
||||
{
|
||||
// Resolve tag to source IDs
|
||||
var matchingIds = sourceDefinitions
|
||||
.Where(s => s.Tags.Contains(value, StringComparer.OrdinalIgnoreCase))
|
||||
.Select(s => s.Id)
|
||||
.OrderBy(id => id, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
if (matchingIds.Count > 0)
|
||||
{
|
||||
resolved["sourceVendor"] = matchingIds;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Split comma-separated values
|
||||
var values = value.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
|
||||
.OrderBy(v => v, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
resolved[key] = values;
|
||||
}
|
||||
}
|
||||
|
||||
return resolved;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Lightweight descriptor for source definitions used by <see cref="MirrorExportOptions.ResolveFilters"/>.
|
||||
/// Decouples Excititor.Core from Concelier.Core while allowing category/tag resolution.
|
||||
/// </summary>
|
||||
public sealed record MirrorSourceDefinitionDescriptor(
|
||||
string Id,
|
||||
string Category,
|
||||
IReadOnlyList<string> Tags);
|
||||
|
||||
public sealed class MirrorSigningOptions
|
||||
{
|
||||
/// <summary>
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace StellaOps.Excititor.Core;
|
||||
@@ -11,7 +12,32 @@ public sealed record MirrorExportPlan(
|
||||
|
||||
public static class MirrorExportPlanner
|
||||
{
|
||||
/// <summary>
|
||||
/// Builds an export plan from the given options, expanding multi-value and
|
||||
/// comma-separated filter values into individual <see cref="VexQueryFilter"/> entries.
|
||||
/// This overload does not resolve <c>sourceCategory</c>/<c>sourceTag</c> shorthands.
|
||||
/// </summary>
|
||||
public static bool TryBuild(MirrorExportOptions exportOptions, out MirrorExportPlan plan, out string? error)
|
||||
=> TryBuild(exportOptions, sourceDefinitions: null, out plan, out error);
|
||||
|
||||
/// <summary>
|
||||
/// Builds an export plan from the given options, expanding multi-value filters,
|
||||
/// comma-separated values, and optionally resolving <c>sourceCategory</c>/<c>sourceTag</c>
|
||||
/// shorthands when <paramref name="sourceDefinitions"/> is provided.
|
||||
/// </summary>
|
||||
/// <param name="exportOptions">The export options containing raw filter configuration.</param>
|
||||
/// <param name="sourceDefinitions">
|
||||
/// Optional source catalog for resolving category/tag shorthands. Pass <c>null</c> to
|
||||
/// skip category/tag resolution (comma-separated expansion still applies).
|
||||
/// </param>
|
||||
/// <param name="plan">The resulting export plan when successful.</param>
|
||||
/// <param name="error">An error code when the build fails.</param>
|
||||
/// <returns><c>true</c> if the plan was built successfully; otherwise <c>false</c>.</returns>
|
||||
public static bool TryBuild(
|
||||
MirrorExportOptions exportOptions,
|
||||
IReadOnlyList<MirrorSourceDefinitionDescriptor>? sourceDefinitions,
|
||||
out MirrorExportPlan plan,
|
||||
out string? error)
|
||||
{
|
||||
if (exportOptions is null)
|
||||
{
|
||||
@@ -35,7 +61,9 @@ public static class MirrorExportPlanner
|
||||
return false;
|
||||
}
|
||||
|
||||
var filters = exportOptions.Filters.Select(pair => new VexQueryFilter(pair.Key, pair.Value));
|
||||
// Resolve filters: expand comma-separated values and category/tag shorthands
|
||||
var resolvedFilters = exportOptions.ResolveFilters(sourceDefinitions);
|
||||
var filters = ExpandResolvedFilters(resolvedFilters);
|
||||
var sorts = exportOptions.Sort.Select(pair => new VexQuerySort(pair.Key, pair.Value));
|
||||
var query = VexQuery.Create(filters, sorts, exportOptions.Limit, exportOptions.Offset, exportOptions.View);
|
||||
var signature = VexQuerySignature.FromQuery(query);
|
||||
@@ -44,4 +72,23 @@ public static class MirrorExportPlanner
|
||||
error = null;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Expands resolved multi-value filters into individual <see cref="VexQueryFilter"/>
|
||||
/// entries. For a key with multiple values (e.g., <c>sourceVendor</c> = ["alpine","debian","ubuntu"]),
|
||||
/// each value produces a separate <see cref="VexQueryFilter"/> with the same key.
|
||||
/// This ensures OR-semantics: a source matches if its vendor ID equals ANY of the values.
|
||||
/// Values are emitted in sorted order for deterministic query signatures.
|
||||
/// </summary>
|
||||
internal static IEnumerable<VexQueryFilter> ExpandResolvedFilters(
|
||||
Dictionary<string, IReadOnlyList<string>> resolvedFilters)
|
||||
{
|
||||
foreach (var (key, values) in resolvedFilters.OrderBy(kvp => kvp.Key, StringComparer.Ordinal))
|
||||
{
|
||||
foreach (var value in values)
|
||||
{
|
||||
yield return new VexQueryFilter(key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,443 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
|
||||
namespace StellaOps.Excititor.Core;
|
||||
|
||||
/// <summary>
|
||||
/// Background service that periodically checks configured mirror domains for stale
|
||||
/// export bundles and triggers regeneration when source data has been updated since
|
||||
/// the last bundle generation. Designed for air-gap awareness: can be fully disabled
|
||||
/// via <see cref="MirrorDistributionOptions.AutoRefreshEnabled"/>.
|
||||
/// </summary>
|
||||
public sealed class MirrorExportScheduler : BackgroundService
|
||||
{
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly IOptionsMonitor<MirrorDistributionOptions> _optionsMonitor;
|
||||
private readonly ILogger<MirrorExportScheduler> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ConcurrentDictionary<string, DomainGenerationStatus> _domainStatus = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of <see cref="MirrorExportScheduler"/>.
|
||||
/// </summary>
|
||||
public MirrorExportScheduler(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
IOptionsMonitor<MirrorDistributionOptions> optionsMonitor,
|
||||
ILogger<MirrorExportScheduler> logger,
|
||||
TimeProvider timeProvider)
|
||||
{
|
||||
_scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory));
|
||||
_optionsMonitor = optionsMonitor ?? throw new ArgumentNullException(nameof(optionsMonitor));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a snapshot of generation status for all tracked domains.
|
||||
/// </summary>
|
||||
public IReadOnlyDictionary<string, DomainGenerationStatus> GetDomainStatuses()
|
||||
=> _domainStatus.ToDictionary(kvp => kvp.Key, kvp => kvp.Value, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
var options = _optionsMonitor.CurrentValue;
|
||||
|
||||
if (!options.AutoRefreshEnabled)
|
||||
{
|
||||
_logger.LogInformation("MirrorExportScheduler is disabled (AutoRefreshEnabled=false). Exiting.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!options.Enabled)
|
||||
{
|
||||
_logger.LogInformation("MirrorExportScheduler is disabled (Mirror distribution disabled). Exiting.");
|
||||
return;
|
||||
}
|
||||
|
||||
var intervalMinutes = Math.Max(options.RefreshIntervalMinutes, 1);
|
||||
|
||||
_logger.LogInformation(
|
||||
"MirrorExportScheduler started. Refresh interval: {IntervalMinutes} min, Domains: {DomainCount}",
|
||||
intervalMinutes,
|
||||
options.Domains.Count);
|
||||
|
||||
// Allow other services to initialize before the first sweep
|
||||
await Task.Delay(TimeSpan.FromSeconds(15), stoppingToken).ConfigureAwait(false);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var currentOptions = _optionsMonitor.CurrentValue;
|
||||
|
||||
if (!currentOptions.AutoRefreshEnabled)
|
||||
{
|
||||
_logger.LogInformation("MirrorExportScheduler auto-refresh disabled at runtime. Stopping refresh loop.");
|
||||
break;
|
||||
}
|
||||
|
||||
if (!currentOptions.Enabled)
|
||||
{
|
||||
_logger.LogDebug("Mirror distribution disabled; skipping refresh cycle.");
|
||||
}
|
||||
else
|
||||
{
|
||||
await RefreshAllDomainsAsync(currentOptions, stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
intervalMinutes = Math.Max(currentOptions.RefreshIntervalMinutes, 1);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Unhandled error in MirrorExportScheduler refresh cycle.");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMinutes(intervalMinutes), stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation("MirrorExportScheduler stopped.");
|
||||
}
|
||||
|
||||
private async Task RefreshAllDomainsAsync(MirrorDistributionOptions options, CancellationToken cancellationToken)
|
||||
{
|
||||
if (options.Domains.Count == 0)
|
||||
{
|
||||
_logger.LogDebug("No mirror domains configured; skipping refresh cycle.");
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogDebug("Starting mirror refresh cycle for {DomainCount} domain(s).", options.Domains.Count);
|
||||
|
||||
foreach (var domain in options.Domains)
|
||||
{
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await RefreshDomainAsync(options, domain, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Failed to refresh mirror domain {DomainId}. Continuing with remaining domains.",
|
||||
domain.Id);
|
||||
|
||||
UpdateDomainStatus(domain.Id, status => status with
|
||||
{
|
||||
LastError = ex.Message,
|
||||
LastErrorAt = _timeProvider.GetUtcNow(),
|
||||
ConsecutiveFailures = status.ConsecutiveFailures + 1,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogDebug("Mirror refresh cycle completed.");
|
||||
}
|
||||
|
||||
private async Task RefreshDomainAsync(
|
||||
MirrorDistributionOptions options,
|
||||
MirrorDomainOptions domain,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (domain.Exports.Count == 0)
|
||||
{
|
||||
_logger.LogDebug("Domain {DomainId} has no exports; skipping.", domain.Id);
|
||||
return;
|
||||
}
|
||||
|
||||
var stalePlanCount = 0;
|
||||
var regeneratedCount = 0;
|
||||
var domainStartTime = Stopwatch.GetTimestamp();
|
||||
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var connectorStateRepo = scope.ServiceProvider.GetService<IVexConnectorStateRepository>();
|
||||
|
||||
// Resolve the latest source update time from connector state
|
||||
DateTimeOffset? latestSourceUpdate = null;
|
||||
if (connectorStateRepo is not null)
|
||||
{
|
||||
var connectorStates = await connectorStateRepo.ListAsync(cancellationToken).ConfigureAwait(false);
|
||||
latestSourceUpdate = connectorStates
|
||||
.Where(s => s.LastUpdated.HasValue)
|
||||
.Max(s => s.LastUpdated);
|
||||
}
|
||||
|
||||
foreach (var exportOption in domain.Exports)
|
||||
{
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (!MirrorExportPlanner.TryBuild(exportOption, out var plan, out var error))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Skipping export {ExportKey} in domain {DomainId}: {Error}",
|
||||
exportOption.Key,
|
||||
domain.Id,
|
||||
error);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var isStale = await CheckExportStalenessAsync(
|
||||
scope.ServiceProvider,
|
||||
plan,
|
||||
latestSourceUpdate,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!isStale)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Export {ExportKey} in domain {DomainId} is fresh; skipping regeneration.",
|
||||
plan.Key,
|
||||
domain.Id);
|
||||
continue;
|
||||
}
|
||||
|
||||
stalePlanCount++;
|
||||
|
||||
_logger.LogInformation(
|
||||
"Export {ExportKey} in domain {DomainId} is stale. Triggering bundle regeneration.",
|
||||
plan.Key,
|
||||
domain.Id);
|
||||
|
||||
var exportStartTime = Stopwatch.GetTimestamp();
|
||||
|
||||
await RegenerateExportAsync(
|
||||
scope.ServiceProvider,
|
||||
options,
|
||||
domain,
|
||||
plan,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var exportDuration = Stopwatch.GetElapsedTime(exportStartTime);
|
||||
regeneratedCount++;
|
||||
|
||||
_logger.LogInformation(
|
||||
"Regenerated export {ExportKey} in domain {DomainId} in {DurationMs:F0}ms.",
|
||||
plan.Key,
|
||||
domain.Id,
|
||||
exportDuration.TotalMilliseconds);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Failed to regenerate export {ExportKey} in domain {DomainId}. Continuing with remaining exports.",
|
||||
plan.Key,
|
||||
domain.Id);
|
||||
}
|
||||
}
|
||||
|
||||
var domainDuration = Stopwatch.GetElapsedTime(domainStartTime);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
if (regeneratedCount > 0)
|
||||
{
|
||||
UpdateDomainStatus(domain.Id, status => status with
|
||||
{
|
||||
LastGeneratedAt = now,
|
||||
LastRefreshDurationMs = domainDuration.TotalMilliseconds,
|
||||
ExportsRegenerated = regeneratedCount,
|
||||
StaleExportsFound = stalePlanCount,
|
||||
ConsecutiveFailures = 0,
|
||||
LastError = null,
|
||||
LastErrorAt = null,
|
||||
});
|
||||
|
||||
_logger.LogInformation(
|
||||
"Domain {DomainId} refresh complete: {Regenerated}/{Stale} exports regenerated in {DurationMs:F0}ms.",
|
||||
domain.Id,
|
||||
regeneratedCount,
|
||||
stalePlanCount,
|
||||
domainDuration.TotalMilliseconds);
|
||||
}
|
||||
else
|
||||
{
|
||||
UpdateDomainStatus(domain.Id, status => status with
|
||||
{
|
||||
LastCheckedAt = now,
|
||||
StaleExportsFound = stalePlanCount,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> CheckExportStalenessAsync(
|
||||
IServiceProvider services,
|
||||
MirrorExportPlan plan,
|
||||
DateTimeOffset? latestSourceUpdate,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// Resolve IVexExportStore from the DI container at runtime to avoid
|
||||
// a compile-time dependency from Excititor.Core on Excititor.Export
|
||||
var exportStore = services.GetService<IMirrorExportManifestLookup>();
|
||||
|
||||
if (exportStore is null)
|
||||
{
|
||||
// If no export store is registered, treat as stale (first run or unconfigured)
|
||||
_logger.LogDebug("No IMirrorExportManifestLookup registered; treating export {ExportKey} as stale.", plan.Key);
|
||||
return true;
|
||||
}
|
||||
|
||||
var lastManifest = await exportStore.FindManifestAsync(plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (lastManifest is null)
|
||||
{
|
||||
// No previous export: definitely stale
|
||||
return true;
|
||||
}
|
||||
|
||||
// If we have source update information, compare against last export time
|
||||
if (latestSourceUpdate.HasValue && lastManifest.CreatedAt < latestSourceUpdate.Value)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Export {ExportKey} created at {ExportCreatedAt:O} is older than latest source update at {SourceUpdate:O}.",
|
||||
plan.Key,
|
||||
lastManifest.CreatedAt,
|
||||
latestSourceUpdate.Value);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static async Task RegenerateExportAsync(
|
||||
IServiceProvider services,
|
||||
MirrorDistributionOptions options,
|
||||
MirrorDomainOptions domain,
|
||||
MirrorExportPlan plan,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var regenerator = services.GetService<IMirrorBundleRegenerator>();
|
||||
|
||||
if (regenerator is null)
|
||||
{
|
||||
// No regenerator registered; bundle generation deferred to the export pipeline
|
||||
return;
|
||||
}
|
||||
|
||||
await regenerator.RegenerateAsync(domain.Id, plan.Key, plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private void UpdateDomainStatus(string domainId, Func<DomainGenerationStatus, DomainGenerationStatus> updater)
|
||||
{
|
||||
_domainStatus.AddOrUpdate(
|
||||
domainId,
|
||||
_ => updater(DomainGenerationStatus.Empty),
|
||||
(_, existing) => updater(existing));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Status record tracking the last generation state for a mirror domain.
|
||||
/// Exposed via <see cref="MirrorExportScheduler.GetDomainStatuses"/> for
|
||||
/// observability and staleness reporting.
|
||||
/// </summary>
|
||||
public sealed record DomainGenerationStatus
|
||||
{
|
||||
public static readonly DomainGenerationStatus Empty = new();
|
||||
|
||||
/// <summary>When the domain was last successfully regenerated.</summary>
|
||||
public DateTimeOffset? LastGeneratedAt { get; init; }
|
||||
|
||||
/// <summary>When the domain was last checked (even if nothing was stale).</summary>
|
||||
public DateTimeOffset? LastCheckedAt { get; init; }
|
||||
|
||||
/// <summary>Duration of the last refresh cycle in milliseconds.</summary>
|
||||
public double LastRefreshDurationMs { get; init; }
|
||||
|
||||
/// <summary>Number of stale exports found in the last cycle.</summary>
|
||||
public int StaleExportsFound { get; init; }
|
||||
|
||||
/// <summary>Number of exports regenerated in the last cycle.</summary>
|
||||
public int ExportsRegenerated { get; init; }
|
||||
|
||||
/// <summary>Number of consecutive domain-level failures.</summary>
|
||||
public int ConsecutiveFailures { get; init; }
|
||||
|
||||
/// <summary>Last error message, if any.</summary>
|
||||
public string? LastError { get; init; }
|
||||
|
||||
/// <summary>When the last error occurred.</summary>
|
||||
public DateTimeOffset? LastErrorAt { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Abstraction for looking up the latest export manifest by signature and format.
|
||||
/// This decouples <see cref="MirrorExportScheduler"/> (Excititor.Core) from the
|
||||
/// concrete <c>IVexExportStore</c> in Excititor.Export without introducing a
|
||||
/// compile-time dependency.
|
||||
/// </summary>
|
||||
public interface IMirrorExportManifestLookup
|
||||
{
|
||||
/// <summary>
|
||||
/// Finds the latest export manifest matching the given query signature and format.
|
||||
/// </summary>
|
||||
ValueTask<MirrorExportManifestSnapshot?> FindManifestAsync(
|
||||
VexQuerySignature signature,
|
||||
VexExportFormat format,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Lightweight snapshot of an export manifest used for staleness comparisons.
|
||||
/// </summary>
|
||||
public sealed record MirrorExportManifestSnapshot(
|
||||
string ExportId,
|
||||
DateTimeOffset CreatedAt,
|
||||
long SizeBytes,
|
||||
int ClaimCount);
|
||||
|
||||
/// <summary>
|
||||
/// Abstraction for triggering bundle regeneration from the scheduler.
|
||||
/// Implemented by the export pipeline (Excititor.Export or WebService layer)
|
||||
/// to perform the actual VEX query, serialization, and bundle publishing.
|
||||
/// </summary>
|
||||
public interface IMirrorBundleRegenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Regenerates the export bundle for the given domain, export key, query signature, and format.
|
||||
/// </summary>
|
||||
Task RegenerateAsync(
|
||||
string domainId,
|
||||
string exportKey,
|
||||
VexQuerySignature signature,
|
||||
VexExportFormat format,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -15,6 +15,7 @@
|
||||
<PackageReference Include="Microsoft.Extensions.Http" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
|
||||
Reference in New Issue
Block a user