diff --git a/docs/modules/concelier/README.md b/docs/modules/concelier/README.md index ccf5099d3..333dcc92f 100644 --- a/docs/modules/concelier/README.md +++ b/docs/modules/concelier/README.md @@ -76,3 +76,4 @@ Concelier ingests signed advisories from **32 advisory connectors** and converts - Sprint 110 attestation chain validated, evidence bundle tests green - Link-Not-Merge cache and console consumption docs frozen - Observation events transport reviewed, NATS/air-gap guidance updated +- Testing-only legacy `AddInMemoryStorage()` compatibility moved into explicit web-service test harnesses, and runtime observation-event defaults no longer imply an undocumented `"inmemory"` transport diff --git a/docs/modules/concelier/architecture.md b/docs/modules/concelier/architecture.md index e2cc35e34..c9ad7cbb1 100644 --- a/docs/modules/concelier/architecture.md +++ b/docs/modules/concelier/architecture.md @@ -109,7 +109,7 @@ Running the same export job twice against the same snapshot must yield byte-iden **Process shape:** single ASP.NET Core service `StellaOps.Concelier.WebService` hosting: -* **Scheduler** with distributed PostgreSQL leases backed by `vuln.job_leases`. Lease coordination is durable; job-run history and internal orchestrator registry state are not yet durably implemented in the live host. +* **Scheduler** with distributed PostgreSQL leases backed by `vuln.job_leases`, durable job-run history in `vuln.job_runs`, and durable internal orchestrator runtime state in `vuln.orchestrator_registry`, `vuln.orchestrator_heartbeats`, `vuln.orchestrator_commands`, and `vuln.orchestrator_manifests`. * **Connectors** (fetch/parse/map) that emit immutable observation candidates. * **Observation writer** enforcing AOC invariants via `AOCWriteGuard`. * **Linkset builder** that correlates observations into `advisory_linksets` and annotates conflicts. @@ -240,12 +240,14 @@ Legacy `Advisory`, `Affected`, and merge-centric entities remain in the reposito ## 4) Source families & precedence -The source catalog contains **75 definitions** across **14 categories**. The authoritative definition lives in `src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceDefinitions.cs`; for the full connector index see `docs/modules/concelier/connectors.md`. +The source catalog contains **78 definitions** across **14 categories**. The authoritative definition lives in `src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceDefinitions.cs`; for the full connector index see `docs/modules/concelier/connectors.md`. + +Runtime surfaces intentionally distinguish between catalog breadth and runnable support. `/api/v1/advisory-sources/catalog` and `/api/v1/advisory-sources/status` expose `syncSupported` plus the resolved `fetchJobKind`, and only sources with a registered fetch pipeline can be enabled or synced through the live host. Operators should use the canonical runtime IDs from the catalog, including `jpcert`, `auscert`, `krcert`, `cert-de`, `adobe`, and `chromium`; legacy aliases such as `jvn`, `acsc`, `kisa`, `cert-bund`, `vndr-adobe`, and `vndr-chromium` remain compatibility fallbacks for configuration binding and source normalization only. ### 3.1 Families * **Primary databases**: NVD, OSV, GHSA, CVE.org (MITRE). -* **Vendor PSIRTs**: Microsoft, Oracle, Cisco, Apple, VMware, Fortinet, Juniper, Palo Alto, plus cloud providers (AWS, Azure, GCP). +* **Vendor PSIRTs**: Red Hat, Microsoft, Oracle, Adobe, Apple, Chromium, Cisco, VMware, Fortinet, Juniper, Palo Alto, plus cloud providers (AWS, Azure, GCP). * **Linux distros**: Debian, Ubuntu, Alpine, SUSE, RHEL, CentOS, Fedora, Arch, Gentoo, Astra Linux. * **OSS ecosystems**: npm, PyPI, Go, RubyGems, NuGet, Maven, Crates.io, Packagist, Hex.pm. * **Package manager native**: RustSec (cargo-audit), PyPA (pip-audit), Go Vuln DB (govulncheck), Ruby Advisory DB (bundler-audit). @@ -429,8 +431,12 @@ Fresh blank databases must enable PostgreSQL extension prerequisites before Conc * TTL index on `occurredAt` (configurable retention), `{type:1, occurredAt:-1}` for replay. * `concelier.export_state` `{_id(exportKind), baseExportId?, baseDigest?, lastFullDigest?, lastDeltaDigest?, cursor, files[]}` -* `job_leases` `{lease_key, holder, acquired_at, heartbeat_at, lease_ms, ttl_at}` used by live scheduler coordination; expired leases can be stolen safely by another runner. -* `jobs` `{_id, type, args, state, startedAt, heartbeatAt, endedAt, error}` is the planned durable owner for future job-run history. Current live runtime does not persist `/jobs` or `/internal/orch/*` state and returns `501` until a durable job/orchestrator registry backend lands. +* `job_leases` `{lease_key, holder, acquired_at, heartbeat_at, lease_ms, ttl_at}` used by live scheduler coordination; expired leases can be stolen safely by another runner. +* `job_runs` `{run_id, kind, status, created_at, started_at, completed_at, trigger, parameters_hash, error, timeout_ms, lease_duration_ms, parameters_json}` stores durable scheduler history surfaced by `/jobs`. +* `orchestrator_registry` `{tenant_id, connector_id, source, capabilities[], auth_ref, schedule_json, rate_policy_json, artifact_kinds[], lock_key, egress_guard_json, created_at, updated_at}` stores durable connector runtime registrations. +* `orchestrator_heartbeats` `{tenant_id, connector_id, run_id, sequence, status, progress, queue_depth, last_artifact_hash, last_artifact_kind, error_code, retry_after_seconds, timestamp_utc}` stores append-only heartbeat history for `/internal/orch/heartbeat`. +* `orchestrator_commands` `{tenant_id, connector_id, run_id, sequence, command, throttle_json, backfill_json, created_at, expires_at}` stores pending internal orchestrator commands with expiry-aware reads. +* `orchestrator_manifests` `{tenant_id, connector_id, run_id, cursor_range_json, artifact_hashes[], dsse_envelope_hash, completed_at}` stores durable completed-run manifests for internal orchestrator consumers. **Legacy tables** (`advisory`, `alias`, `affected`, `reference`, `merge_event`) remain read-only during the migration window to support back-compat exports. New code must not write to them; scheduled cleanup removes them after Link-Not-Merge GA. @@ -491,9 +497,17 @@ POST /sources/{name}/pause | /resume → toggle GET /jobs/{id} → job status ``` -Current runtime note: `/jobs`, `/internal/orch/*`, and the coordinator-backed manual sync compatibility routes (`/api/v1/advisory-sources/{sourceId}/sync`, `/api/v1/advisory-sources/sync`, `/api/v1/concelier/mirrors/{mirrorId}/sync`) are not durably implemented in the live host. Outside `Testing` they return explicit `501` responses rather than falling back to in-memory state. +Current runtime note: `/jobs`, `/internal/orch/*`, and the coordinator-backed manual sync compatibility routes (`/api/v1/advisory-sources/{sourceId}/sync`, `/api/v1/advisory-sources/sync`, `/api/v1/concelier/mirrors/{mirrorId}/sync`) now run against durable PostgreSQL-backed runtime services (`PostgresJobStore`, `JobCoordinator`, and `PostgresOrchestratorRegistryStore`) in the live host. Process-local placeholder implementations remain limited to `Testing` harnesses. Restart-safe endpoint verification now covers persisted job runs, orchestrator registry records, heartbeats, and queued commands through the live HTTP surface, and the obsolete `UnsupportedJobCoordinator` / `UnsupportedOrchestratorRegistryStore` live-host guards have been removed. See `SPRINT_20260419_029_Concelier_durable_jobs_orchestrator_runtime` for the cutover record and `SPRINT_20260419_030_Concelier_durable_runtime_endpoint_verification` for the endpoint proof. Current signals note: `/v1/signals/symbols/*` is backed by a durable PostgreSQL affected-symbol store (`PostgresAffectedSymbolStore`). Advisory observations are persisted via `PostgresAdvisoryObservationStore`, which invokes the registered `IAffectedSymbolExtractor` chain during `UpsertAsync` to project ingest-time symbols into the affected-symbol store. The legacy non-testing `501` fallback has been removed; in-memory implementations remain only for `Testing` environments. See `SPRINT_20260419_027_Concelier_durable_affected_symbol_runtime` for the cutover record. -Current advisory source note: the live host also exposes `/api/v1/advisory-sources/*` for operator/UI source status, enablement, health checks, and sync triggers. Enabled state is now persisted in the Concelier source store so restarts, setup skip/apply flows, and later integrations-page toggles all observe the same source truth. Platform setup uses bootstrap-key-protected `/internal/setup/advisory-sources/{probe,apply}` endpoints to seed initial source configuration without requiring a tenant session. +Current advisory source note: the live host also exposes `/api/v1/advisory-sources/*` for operator/UI source status, enablement, health checks, sync triggers, and visible review counts. Enabled state is now persisted in the Concelier source store so restarts, setup skip/apply flows, and later integrations-page toggles all observe the same source truth. Advisory totals and signature rollups are projected from distinct `source_advisory_id` values in `vuln.advisory_source_edge`, so the freshness surface reflects the canonical ingest path rather than the legacy `vuln.advisories.source_id` compatibility rows. Platform setup uses bootstrap-key-protected `/internal/setup/advisory-sources/{probe,apply}` endpoints to seed initial source configuration without requiring a tenant session. + +The operator-facing review counters exposed by `/api/v1/advisory-sources` now have fixed semantics: +- `sourceDocumentCount` and the backward-compatible `totalAdvisories` alias are both the count of distinct upstream `source_advisory_id` values linked to the source. +- `canonicalAdvisoryCount` is the count of distinct canonical advisory ids reached through `vuln.advisory_source_edge`. +- `cveCount` is the count of distinct non-empty canonical CVE ids associated with those linked advisories. +- `vexDocumentCount` is the count of distinct linked source advisory documents carrying non-null vendor-status/VEX statements. + +Web operators should derive visible summary cards and per-source detail panels from this same advisory-source contract so Integrations, Security, and setup review screens do not drift in headline counts or terminology. Mirror bootstrap truthfulness note: `/internal/setup/advisory-sources/{probe,apply}` now validates the selected mirror/source configuration before apply succeeds. Mirror mode probes `/concelier/exports/index.json` with normal runtime TLS rules, so certificate/hostname mismatches are returned as actionable setup failures instead of "background sync" warnings. **Exports** @@ -533,9 +547,11 @@ DELETE /domains/{domainId}/exports/{exportKey} → remove an export from a domai POST /domains/{domainId}/generate → trigger on-demand bundle generation GET /domains/{domainId}/status → domain sync status (last generate, staleness) POST /test → test mirror endpoint connectivity -``` - -**Mirror consumer configuration** (under `/api/v1/advisory-sources/mirror`) +``` + +Mirror domain `sourceIds` are validated against the registered advisory source catalog exposed by `/api/v1/advisory-sources/catalog`. Operators can create domains from catalog keys such as `nvd` and `osv` even on a fresh database before any source-status workflow has persisted rows into `vuln.sources`; unknown keys are still rejected. + +**Mirror consumer configuration** (under `/api/v1/advisory-sources/mirror`) ``` GET /consumer → current consumer connector configuration (base address, domain, signature, timeout, connection status, last sync) diff --git a/docs/modules/concelier/connectors.md b/docs/modules/concelier/connectors.md index 3b287f418..40db04ea6 100644 --- a/docs/modules/concelier/connectors.md +++ b/docs/modules/concelier/connectors.md @@ -2,7 +2,18 @@ This index lists Concelier connectors, their status, authentication expectations, and links to operational runbooks. For procedures and alerting, see `docs/modules/concelier/operations/connectors/`. -The catalog currently contains **75 source definitions** across **14 categories**. The authoritative source list is defined in `src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceDefinitions.cs`. +Operator configuration note: + +- Supported advisory source credentials and endpoint overrides can now be supplied through the Web UI or `stella db connectors configure ...`. +- GHSA, Cisco, and Microsoft use operator-supplied credentials through that path. +- Oracle, Adobe, and Chromium use public defaults and only need UI or CLI input when you override or mirror the upstream endpoints. +- See [source-credentials.md](docs/modules/concelier/operations/source-credentials.md). + +The catalog currently contains **78 source definitions** across **14 categories**. The authoritative source list is defined in `src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceDefinitions.cs`. + +Canonical runtime note: the operator-facing source IDs in this index are the only scheduler/catalog IDs that should be used for Concelier jobs and setup. Legacy connector aliases such as `ics-cisa`, `ics-kaspersky`, `ru-bdu`, `ru-nkcki`, `vndr-adobe`, `vndr-apple`, `vndr-chromium`, `vndr-cisco`, `vndr-oracle`, and `vndr.msrc` remain compatibility-only aliases inside normalization paths and must not appear as primary runtime job keys. + +Runtime note: the Concelier advisory catalog and the Excititor default VEX mirror bootstrap share some upstream vendors but are not the same pipeline. The default public VEX bootstrap currently seeds only `redhat`, `ubuntu`, `oracle`, and `cisco`, uses their public CSAF/notice endpoints, and staggers initial runs (`5m`, `7m`, `9m`, `11m`) to avoid burst-fetching multiple upstreams at the same instant. --- @@ -12,7 +23,7 @@ The catalog currently contains **75 source definitions** across **14 categories* | --- | --- | --- | | Primary | Core vulnerability databases (NVD, OSV, GHSA, CVE) | 4 | | Threat | Threat intelligence, exploit prediction, and known-exploited (EPSS, KEV, MITRE ATT&CK, D3FEND) | 4 | -| Vendor | Vendor PSIRTs and cloud provider security bulletins | 14 | +| Vendor | Vendor PSIRTs and cloud provider security bulletins | 16 | | Distribution | Linux distribution security trackers | 10 | | Ecosystem | Language-ecosystem advisory feeds via OSV/GHSA | 9 | | PackageManager | Native package manager advisory databases (cargo-audit, pip-audit, govulncheck, bundler-audit) | 4 | @@ -21,7 +32,7 @@ The catalog currently contains **75 source definitions** across **14 categories* | Container | Container image advisory sources | 2 | | Hardware | Hardware and firmware PSIRT advisories | 3 | | Ics | Industrial control systems and SCADA advisories | 2 | -| Cert | National CERTs and government CSIRTs | 13 | +| Cert | National CERTs and government CSIRTs | 15 | | Mirror | StellaOps pre-aggregated mirrors | 1 | | Other | Uncategorized sources | 0 | @@ -52,11 +63,13 @@ MITRE ATT&CK provides adversary tactics and techniques in STIX format from the ` | Connector | Source ID | Status | Auth | Priority | Ops Runbook | | --- | --- | --- | --- | --- | --- | | Red Hat Security | `redhat` | stable | none | 30 | [redhat.md](docs/modules/concelier/operations/connectors/redhat.md) | -| Microsoft Security (MSRC) | `microsoft` | stable | none | 35 | [msrc.md](docs/modules/concelier/operations/connectors/msrc.md) | +| Microsoft Security (MSRC) | `microsoft` | stable | oauth | 35 | [msrc.md](docs/modules/concelier/operations/connectors/msrc.md) | | Amazon Linux Security | `amazon` | stable | none | 40 | -- | | Google Security | `google` | stable | none | 45 | -- | | Oracle Security | `oracle` | stable | none | 50 | [oracle.md](docs/modules/concelier/operations/connectors/oracle.md) | +| Adobe Security | `adobe` | stable | none | 52 | [adobe.md](docs/modules/concelier/operations/connectors/adobe.md) | | Apple Security | `apple` | stable | none | 55 | [apple.md](docs/modules/concelier/operations/connectors/apple.md) | +| Chromium Stable Channel Updates | `chromium` | stable | none | 57 | [chromium.md](docs/modules/concelier/operations/connectors/chromium.md) | | Cisco Security | `cisco` | stable | oauth | 60 | [cisco.md](docs/modules/concelier/operations/connectors/cisco.md) | | Fortinet PSIRT | `fortinet` | stable | none | 65 | -- | | Juniper Security | `juniper` | stable | none | 70 | -- | @@ -68,6 +81,11 @@ MITRE ATT&CK provides adversary tactics and techniques in STIX format from the ` AWS, Azure, and GCP cloud provider advisories were added in Sprint 007. They track platform-level security bulletins for cloud infrastructure components and are categorized under `Vendor` alongside traditional PSIRTs. +Mirror bootstrap note: +- `oracle` default VEX bootstrap discovery uses Oracle's public security RSS feed and derived `*csaf.json` documents. +- `cisco` default VEX bootstrap uses Cisco's public CSAF provider metadata and does not require the OAuth credentials used by the Concelier openVuln connector. +- If Cisco's public paged catalog is unavailable, the bootstrap falls back to `changes.csv` and then `index.txt`, prefers newer candidates first, and checkpoints seen or permanently inaccessible legacy paths so hourly runs do not re-download or stall on the full historical corpus. + ## Linux Distributions | Connector | Source ID | Status | Auth | Priority | Regions | Ops Runbook | @@ -83,6 +101,9 @@ AWS, Azure, and GCP cloud provider advisories were added in Sprint 007. They tra | Gentoo Security | `gentoo` | stable | none | 46 | -- | -- | | Astra Linux Security | `astra` | stable | none | 48 | RU, CIS | [astra.md](docs/modules/concelier/operations/connectors/astra.md) | +Mirror bootstrap note: +- `ubuntu` default VEX bootstrap reads `https://ubuntu.com/security/notices.json` and synthesizes deterministic CSAF documents from the per-notice JSON payloads because Canonical's public path is notice JSON rather than native CSAF. + ## Language Ecosystems | Connector | Source ID | Status | Auth | Priority | Ops Runbook | @@ -166,15 +187,17 @@ Industrial control systems advisories cover SCADA and operational technology vul | CERT.be (Belgium) | `cert-be` | stable | none | 86 | BE, EU | -- | | NCSC-CH (Switzerland) | `cert-ch` | stable | none | 88 | CH | -- | | CERT-EU | `cert-eu` | stable | none | 90 | EU | -- | +| CCCS (Canada) | `cccs` | stable | none | 91 | CA, NA | [cccs.md](docs/modules/concelier/operations/connectors/cccs.md) | | JPCERT/CC (Japan) | `jpcert` | stable | none | 92 | JP, APAC | [jvn.md](docs/modules/concelier/operations/connectors/jvn.md) | -| CISA (US-CERT) | `us-cert` | stable | none | 94 | US, NA | [cert-cc.md](docs/modules/concelier/operations/connectors/cert-cc.md) | +| CERT/CC | `cert-cc` | stable | none | 93 | US, NA | [cert-cc.md](docs/modules/concelier/operations/connectors/cert-cc.md) | +| CISA (US-CERT) | `us-cert` | stable | none | 94 | US, NA | [ics-cisa.md](docs/modules/concelier/operations/connectors/ics-cisa.md) | | CERT-UA (Ukraine) | `cert-ua` | stable | none | 95 | UA | -- | | CERT.PL (Poland) | `cert-pl` | stable | none | 96 | PL, EU | -- | | AusCERT (Australia) | `auscert` | stable | none | 97 | AU, APAC | -- | -| KrCERT/CC (South Korea) | `krcert` | stable | none | 98 | KR, APAC | -- | +| KrCERT/CC (South Korea) | `krcert` | stable | none | 98 | KR, APAC | [kisa.md](docs/modules/concelier/operations/connectors/kisa.md) | | CERT-In (India) | `cert-in` | stable | none | 99 | IN, APAC | [cert-in.md](docs/modules/concelier/operations/connectors/cert-in.md) | -Five additional CERTs were added in Sprint 007: CERT-UA, CERT.PL, AusCERT, KrCERT/CC, and CERT-In, extending coverage to Eastern Europe, Oceania, and South/East Asia. +Seven additional CERTs beyond the original European/Japanese set are now defined in the catalog: CCCS (Canada), CERT/CC, CERT-UA, CERT.PL, AusCERT, KrCERT/CC, and CERT-In, extending coverage to North America, Eastern Europe, Oceania, and South/East Asia. ## Russian/CIS Sources diff --git a/docs/modules/concelier/link-not-merge-schema.md b/docs/modules/concelier/link-not-merge-schema.md index 0055fff6f..54099d3ce 100644 --- a/docs/modules/concelier/link-not-merge-schema.md +++ b/docs/modules/concelier/link-not-merge-schema.md @@ -19,7 +19,7 @@ _Frozen v1 (add-only) — approved 2025-11-17 for CONCELIER-LNM-21-001/002/101._ "properties": { "_id": {"bsonType": "objectId"}, "tenantId": {"bsonType": "string"}, - "source": {"bsonType": "string", "description": "Adapter id, e.g., ghsa, nvd, cert-bund"}, + "source": {"bsonType": "string", "description": "Adapter id, e.g., ghsa, nvd, cert-de"}, "advisoryId": {"bsonType": "string"}, "title": {"bsonType": "string"}, "summary": {"bsonType": "string"}, diff --git a/docs/modules/concelier/operations/connectors/acsc.md b/docs/modules/concelier/operations/connectors/acsc.md index bee314c5a..cadaf65f3 100644 --- a/docs/modules/concelier/operations/connectors/acsc.md +++ b/docs/modules/concelier/operations/connectors/acsc.md @@ -3,7 +3,7 @@ _Last updated: 2026-01-16_ ## 1. Overview -The ACSC connector ingests Australian Cyber Security Centre advisories and maps them to canonical IDs. +The ACSC connector ingests Australian Cyber Security Centre advisories and maps them to canonical IDs. The runtime/catalog source ID is `auscert`. ## 2. Authentication - No authentication required for public feeds. @@ -12,13 +12,15 @@ The ACSC connector ingests Australian Cyber Security Centre advisories and maps ```yaml concelier: sources: - acsc: + auscert: baseUri: "" maxDocumentsPerFetch: 20 fetchTimeout: "00:00:45" requestDelay: "00:00:00" ``` +Legacy `concelier:sources:acsc` binding is still accepted for compatibility, but new setups should use `concelier:sources:auscert` so the documented defaults match `/api/v1/advisory-sources`. + ## 4. Offline and air-gapped deployments - Mirror ACSC feeds into the Offline Kit and repoint `baseUri` to the mirror. diff --git a/docs/modules/concelier/operations/connectors/adobe.md b/docs/modules/concelier/operations/connectors/adobe.md index 1817e18b9..5af5edbbc 100644 --- a/docs/modules/concelier/operations/connectors/adobe.md +++ b/docs/modules/concelier/operations/connectors/adobe.md @@ -1,26 +1,49 @@ # Concelier Adobe PSIRT Connector - Operations Runbook -_Last updated: 2026-01-16_ +_Last updated: 2026-04-22_ ## 1. Overview -The Adobe connector ingests Adobe PSIRT advisories and maps them to canonical IDs. + +The Adobe connector ingests Adobe PSIRT advisories and maps them to canonical IDs. The canonical runtime source ID is `adobe`. ## 2. Authentication + - No authentication required for public advisories. -## 3. Configuration (`concelier.yaml`) +## 3. Configuration paths + +Primary operator path: + +- Web UI: **Security Posture -> Configure Sources** or **Ops -> Operations -> Feeds & Airgap -> Configure Sources** +- CLI: + ```bash + stella db connectors configure adobe \ + --server https://concelier.example.internal \ + --set indexUri=https://mirror.example.internal/adobe/security-bulletin.html + ``` + +The Adobe connector does not require credentials. Use the UI/CLI configuration path only when overriding the canonical Adobe public index or supplying additional mirror or index URIs. + +Compatibility fallback (`concelier.yaml`): + ```yaml concelier: sources: adobe: - baseUri: "" - maxDocumentsPerFetch: 20 - fetchTimeout: "00:00:45" - requestDelay: "00:00:00" + indexUri: "https://helpx.adobe.com/security/security-bulletin.html" + additionalIndexUris: [] + initialBackfill: "90.00:00:00" + windowOverlap: "3.00:00:00" + maxEntriesPerFetch: 100 ``` ## 4. Offline and air-gapped deployments -- Mirror advisories into the Offline Kit and repoint `baseUri` to the mirror. + +- Mirror the Adobe index and detail pages into the Offline Kit. +- Repoint `indexUri` and `additionalIndexUris` to the mirrored allowlisted endpoints. ## 5. Common failure modes -- Upstream format changes or delayed bulletin updates. + +- Adobe changes the bulletin index HTML layout or detail-table structure +- Historic bulletin pages disappear or move without redirects +- Operators mirror detail pages but forget to mirror the index page that seeds discovery diff --git a/docs/modules/concelier/operations/connectors/certbund.md b/docs/modules/concelier/operations/connectors/certbund.md index 46ae5fa90..e5203d39c 100644 --- a/docs/modules/concelier/operations/connectors/certbund.md +++ b/docs/modules/concelier/operations/connectors/certbund.md @@ -1,8 +1,10 @@ # Concelier CERT-Bund Connector Operations -_Last updated: 2025-10-17_ +_Last updated: 2025-10-17_ + +Runtime/catalog source ID: `cert-de`. Legacy `cert-bund` remains a compatibility-only configuration alias. -Germany’s Federal Office for Information Security (BSI) operates the Warn- und Informationsdienst (WID) portal. The Concelier CERT-Bund connector (`source:cert-bund:*`) ingests the public RSS feed, hydrates the portal’s JSON detail endpoint, and maps the result into canonical advisories while preserving the original German content. +Germany’s Federal Office for Information Security (BSI) operates the Warn- und Informationsdienst (WID) portal. The Concelier CERT-Bund connector uses the runtime/catalog source ID `cert-de`, so the live fetch pipeline runs as `source:cert-de:*` while preserving the original German content. --- @@ -17,12 +19,12 @@ Germany’s Federal Office for Information Security (BSI) operates the Warn- und Example `concelier.yaml` fragment: ```yaml -concelier: - sources: - cert-bund: - feedUri: "https://wid.cert-bund.de/content/public/securityAdvisory/rss" - portalBootstrapUri: "https://wid.cert-bund.de/portal/" - detailApiUri: "https://wid.cert-bund.de/portal/api/securityadvisory" +concelier: + sources: + cert-de: + feedUri: "https://wid.cert-bund.de/content/public/securityAdvisory/rss" + portalBootstrapUri: "https://wid.cert-bund.de/portal/" + detailApiUri: "https://wid.cert-bund.de/portal/api/securityadvisory" maxAdvisoriesPerFetch: 50 maxKnownAdvisories: 512 requestTimeout: "00:00:30" @@ -59,7 +61,7 @@ Alerting ideas: 1. `increase(certbund.detail.fetch.failures_total[10m]) > 0` 2. `rate(certbund.map.success_total[30m]) == 0` -3. `histogram_quantile(0.95, rate(concelier_source_http_duration_bucket{concelier_source="cert-bund"}[15m])) > 5s` +3. `histogram_quantile(0.95, rate(concelier_source_http_duration_bucket{concelier_source="cert-de"}[15m])) > 5s` The WebService now registers the meter so metrics surface automatically once OpenTelemetry metrics are enabled. @@ -124,7 +126,7 @@ operating offline. ### 3.4 Connector-driven catch-up 1. Temporarily raise `maxAdvisoriesPerFetch` (e.g. 150) and reduce `requestDelay`. -2. Run `stella db fetch --source cert-bund --stage fetch`, then `--stage parse`, then `--stage map` until the fetch log reports `enqueued=0`. +2. Run `stella db fetch --source cert-de --stage fetch`, then `--stage parse`, then `--stage map` until the fetch log reports `enqueued=0`. 3. Restore defaults and capture the cursor snapshot for audit. --- diff --git a/docs/modules/concelier/operations/connectors/chromium.md b/docs/modules/concelier/operations/connectors/chromium.md index 9d7f0ec53..60a66a5f8 100644 --- a/docs/modules/concelier/operations/connectors/chromium.md +++ b/docs/modules/concelier/operations/connectors/chromium.md @@ -1,26 +1,49 @@ # Concelier Chromium Connector - Operations Runbook -_Last updated: 2026-01-16_ +_Last updated: 2026-04-22_ ## 1. Overview -The Chromium connector ingests Chromium security advisories and maps them to canonical IDs. + +The Chromium connector ingests Chromium security advisories and maps them to canonical IDs. The canonical runtime source ID is `chromium`. ## 2. Authentication + - No authentication required for public advisories. -## 3. Configuration (`concelier.yaml`) +## 3. Configuration paths + +Primary operator path: + +- Web UI: **Security Posture -> Configure Sources** or **Ops -> Operations -> Feeds & Airgap -> Configure Sources** +- CLI: + ```bash + stella db connectors configure chromium \ + --server https://concelier.example.internal \ + --set feedUri=https://mirror.example.internal/chromium/atom.xml + ``` + +The Chromium connector does not require credentials. Use the UI/CLI configuration path only when overriding the canonical Chrome Releases Atom feed for a mirror or controlled ingestion path. + +Compatibility fallback (`concelier.yaml`): + ```yaml concelier: sources: chromium: - baseUri: "" - maxDocumentsPerFetch: 20 - fetchTimeout: "00:00:45" - requestDelay: "00:00:00" + feedUri: "https://chromereleases.googleblog.com/atom.xml" + initialBackfill: "30.00:00:00" + windowOverlap: "2.00:00:00" + maxFeedPages: 4 + maxEntriesPerPage: 50 ``` ## 4. Offline and air-gapped deployments -- Mirror advisories into the Offline Kit and repoint `baseUri` to the mirror. + +- Mirror the Atom feed and referenced post pages into the Offline Kit. +- Repoint `feedUri` to the mirrored allowlisted endpoint. ## 5. Common failure modes -- Feed cadence shifts during Chromium release trains. + +- Feed cadence shifts during Chromium release trains +- Google changes the Atom feed or post markup used for stable-channel parsing +- Operators mirror post pages but not the Atom feed that seeds discovery diff --git a/docs/modules/concelier/operations/connectors/cisco.md b/docs/modules/concelier/operations/connectors/cisco.md index 7284e8f9c..a140f166d 100644 --- a/docs/modules/concelier/operations/connectors/cisco.md +++ b/docs/modules/concelier/operations/connectors/cisco.md @@ -1,94 +1,99 @@ -# Concelier Cisco PSIRT Connector – OAuth Provisioning SOP - -_Last updated: 2025-10-14_ - -## 1. Scope - -This runbook describes how Ops provisions, rotates, and distributes Cisco PSIRT openVuln OAuth client credentials for the Concelier Cisco connector. It covers online and air-gapped (Offline Kit) environments, quota-aware execution, and escalation paths. - -## 2. Prerequisites - -- Active Cisco.com (CCO) account with access to the Cisco API Console. -- Cisco PSIRT openVuln API entitlement (visible under “My Apps & Keys” once granted).citeturn3search0 -- Concelier configuration location (typically `/etc/stella/concelier.yaml` in production) or Offline Kit secret bundle staging directory. - -## 3. Provisioning workflow - -1. **Register the application** - - Sign in at . - - Select **Register a New App** → Application Type: `Service`, Grant Type: `Client Credentials`, API: `Cisco PSIRT openVuln API`.citeturn3search0 - - Record the generated `clientId` and `clientSecret` in the Ops vault. -2. **Verify token issuance** - - Request an access token with: - ```bash - curl -s https://id.cisco.com/oauth2/default/v1/token \ - -H "Content-Type: application/x-www-form-urlencoded" \ - -d "grant_type=client_credentials" \ - -d "client_id=${CLIENT_ID}" \ - -d "client_secret=${CLIENT_SECRET}" - ``` - - Confirm HTTP 200 and an `expires_in` value of 3600 seconds (tokens live for one hour).citeturn3search0turn3search7 - - Preserve the response only long enough to validate syntax; do **not** persist tokens. -3. **Authorize Concelier runtime** - - Update `concelier:sources:cisco:auth` (or the module-specific secret template) with the stored credentials. - - For Offline Kit delivery, export encrypted secrets into `offline-kit/secrets/cisco-openvuln.json` using the platform’s sealed secret format. -4. **Connectivity validation** - - From the Concelier control plane, run `stella db fetch --source vndr-cisco --stage fetch` (use staging or a controlled window). - - Ensure the Source HTTP diagnostics record `Bearer` authorization headers and no 401/403 responses. - -## 4. Rotation SOP - -| Step | Owner | Notes | -| --- | --- | --- | -| 1. Schedule rotation | Ops (monthly board) | Rotate every 90 days or immediately after suspected credential exposure. | -| 2. Create replacement app | Ops | Repeat §3.1 with “-next” suffix; verify token issuance. | -| 3. Stage dual credentials | Ops + Concelier On-Call | Publish new credentials to secret store alongside current pair. | -| 4. Cut over | Concelier On-Call | Restart connector workers during a low-traffic window (<10 min) to pick up the new secret. | -| 5. Deactivate legacy app | Ops | Delete prior app in Cisco API Console once telemetry confirms successful fetch/parse cycles for 2 consecutive hours. | - -**Automation hooks** -- Rotation reminders are tracked in OpsRunbookOps board (`OPS-RUN-KEYS` swim lane); add checklist items for Concelier Cisco when opening a rotation task. -- Use the secret management pipeline (`ops/secrets/rotate.sh --connector cisco`) to template vault updates; the script renders a redacted diff for audit. - -## 5. Offline Kit packaging - -1. Generate the credential bundle using the Offline Kit CLI: - `offline-kit secrets add cisco-openvuln --client-id … --client-secret …` -2. Store the encrypted payload under `offline-kit/secrets/cisco-openvuln.enc`. -3. Distribute via the Offline Kit channel; update `offline-kit/MANIFEST.md` with the credential fingerprint (SHA256 of plaintext concatenated with metadata). -4. Document validation steps for the receiving site (token request from an air-gapped relay or cached token mirror). - -## 6. Quota and throttling guidance - -- Cisco enforces combined limits of 5 requests/second, 30 requests/minute, and 5 000 requests/day per application.citeturn0search0turn3search6 -- Concelier fetch jobs must respect `Retry-After` headers on HTTP 429 responses; Ops should monitor for sustained quota saturation and consider paging window adjustments. -- Telemetry to watch: `concelier.source.http.requests{concelier.source="vndr-cisco"}`, `concelier.source.http.failures{...}`, and connector-specific metrics once implemented. - -## 7. Telemetry & Monitoring - -- **Metrics (Meter `StellaOps.Concelier.Connector.Vndr.Cisco`)** - - `cisco.fetch.documents`, `cisco.fetch.failures`, `cisco.fetch.unchanged` - - `cisco.parse.success`, `cisco.parse.failures` - - `cisco.map.success`, `cisco.map.failures`, `cisco.map.affected.packages` -- **Shared HTTP metrics** via `SourceDiagnostics`: - - `concelier.source.http.requests{concelier.source="vndr-cisco"}` - - `concelier.source.http.failures{concelier.source="vndr-cisco"}` - - `concelier.source.http.duration{concelier.source="vndr-cisco"}` -- **Structured logs** - - `Cisco fetch completed date=… pages=… added=…` (info) - - `Cisco parse completed parsed=… failures=…` (info) - - `Cisco map completed mapped=… failures=…` (info) - - Warnings surface when DTO serialization fails or document payload is missing. -- Suggested alerts: non-zero `cisco.fetch.failures` in 15m, or `cisco.map.success` flatlines while fetch continues. - -## 8. Incident response - -- **Token compromise** – revoke the application in the Cisco API Console, purge cached secrets, rotate immediately per §4. -- **Persistent 401/403** – confirm credentials in vault, then validate token issuance; if unresolved, open a Cisco DevNet support ticket referencing the application ID. -- **429 spikes** – inspect job scheduler cadence and adjust connector options (`maxRequestsPerWindow`) before requesting higher quotas from Cisco. - -## 9. References - -- Cisco PSIRT openVuln API Authentication Guide.citeturn3search0 -- Accessing the openVuln API using curl (token lifetime).citeturn3search7 -- openVuln API rate limit documentation.citeturn0search0turn3search6 +# Concelier Cisco PSIRT Connector - OAuth Provisioning SOP + +_Last updated: 2026-04-22_ + +## 0. Scope note + +This runbook covers the authenticated Concelier Cisco openVuln advisory connector. + +It does not describe the default Excititor public VEX mirror bootstrap. The default VEX bootstrap path remains credential-free and uses Cisco public CSAF metadata and documents. + +## 1. Scope + +This runbook describes how Ops provisions, rotates, and distributes Cisco PSIRT openVuln OAuth client credentials for the Concelier Cisco connector. It covers online and air-gapped environments, quota-aware execution, and escalation paths. + +Primary operator path: + +- Web UI: **Security Posture -> Configure Sources** or **Ops -> Operations -> Feeds & Airgap -> Configure Sources** +- CLI: + ```bash + stella db connectors configure cisco \ + --server https://concelier.example.internal \ + --set clientId=... \ + --set clientSecret=... + ``` + +Persisted source configuration is now the preferred operator path. Host-local YAML and external secret distribution remain compatibility fallbacks for older installs. + +## 2. Prerequisites + +- Active Cisco.com account with access to the Cisco API Console +- Cisco PSIRT openVuln API entitlement visible under **My Apps & Keys** +- Concelier control-plane access + +Entitlement notes: + +- There is no separate StellaOps-side fee for this connector. +- Cisco still requires a Cisco.com account, API Console registration, and visible access to the PSIRT openVuln API entry. +- If the openVuln API is not visible for the account, the connector cannot be fully activated until Cisco-side entitlement is in place. + +## 3. Provisioning workflow + +1. Register the application + - Sign in at `https://apiconsole.cisco.com`. + - Open **My Apps & Keys**. + - Click **Register a New App**. + - Choose: + - Application Type: `Service` + - Grant Type: `Client Credentials` + - API: `Cisco PSIRT openVuln API` + - Record the generated `clientId` and `clientSecret`. +2. Verify token issuance + - Request an access token: + ```bash + curl -s https://id.cisco.com/oauth2/default/v1/token \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "grant_type=client_credentials" \ + -d "client_id=${CLIENT_ID}" \ + -d "client_secret=${CLIENT_SECRET}" + ``` + - Confirm HTTP 200 and a usable `expires_in` value. + - Do not persist the token itself. +3. Authorize Concelier runtime + - Store the `clientId` and `clientSecret` in StellaOps through the Web UI or the CLI command shown above. +4. Connectivity validation + - From the Concelier control plane, run: + ```bash + stella db fetch --source cisco --stage fetch + ``` + - Confirm no 401/403 responses in source diagnostics. + +## 4. Rotation SOP + +| Step | Owner | Notes | +| --- | --- | --- | +| 1. Schedule rotation | Ops | Rotate every 90 days or immediately after suspected exposure. | +| 2. Create replacement app | Ops | Repeat the registration flow and verify token issuance. | +| 3. Stage new credentials | Ops + Concelier on-call | Store the replacement values in StellaOps and verify a staged fetch. | +| 4. Cut over | Concelier on-call | Validate successful fetch/parse cycles during a low-traffic window. | +| 5. Deactivate legacy app | Ops | Delete the old app in Cisco API Console after successful validation. | + +## 5. Offline Kit packaging + +If the site still mirrors secrets out of band: + +1. Generate the credential bundle using the Offline Kit CLI. +2. Store the encrypted payload under the site-approved secret bundle location. +3. Record the delivery fingerprint in the manifest. + +## 6. Quota and throttling guidance + +- Cisco documents combined limits for requests per second, minute, and day per application. +- Concelier must respect `Retry-After` on HTTP 429 responses. +- If quota saturation persists, adjust job cadence before requesting higher quotas from Cisco. + +## 7. Incident response + +- Token compromise: revoke the application in Cisco API Console, purge retained credentials, rotate immediately. +- Persistent 401/403: verify the stored credentials, then verify token issuance directly against Cisco. +- 429 spikes: inspect scheduler cadence and source tuning before escalating upstream. diff --git a/docs/modules/concelier/operations/connectors/cve-kev.md b/docs/modules/concelier/operations/connectors/cve-kev.md index 00b955d04..c7a003213 100644 --- a/docs/modules/concelier/operations/connectors/cve-kev.md +++ b/docs/modules/concelier/operations/connectors/cve-kev.md @@ -115,9 +115,32 @@ Treat repeated schema failures or growing anomaly counts as an upstream regressi - `kev.map.advisories` (tag `catalogVersion`) 4. Confirm `concelier.source.http.requests_total{concelier_source="kev"}` increments once per fetch and that the paired `concelier.source.http.failures_total` stays flat (zero increase). 5. Inspect the info logs `Fetched KEV catalog document … pendingDocuments=…` and `Parsed KEV catalog document … entries=…`—they should appear exactly once per run and `Mapped X/Y… skipped=0` should match the `kev.map.advisories` delta. -6. Confirm PostgreSQL records exist for the catalog JSON (`raw_documents` and `dtos` tables in schema `vuln`) and that advisories with prefix `kev/` are written. - -### 2.4 Production Monitoring +6. Confirm PostgreSQL records exist for the catalog JSON (`raw_documents` and `dtos` tables in schema `vuln`) and that advisories with prefix `kev/` are written. + +### 2.3.1 Cached Replay Recovery + +Use this recovery when KEV fetches already succeeded against an unchanged upstream catalog, but a mapper or persistence fix needs to rebuild `source_id` bindings or canonical/source-edge projections from the latest cached catalog. + +1. Trigger a forced parse replay against the latest cached KEV document: + - `POST /jobs/source:kev:parse` + - Body: `{"trigger":"api","parameters":{"force":"true"}}` +2. Wait for the parse run to finish with `status=Succeeded` via `GET /jobs/{runId}`. +3. Trigger a forced map replay against the same cached document set: + - `POST /jobs/source:kev:map` + - Body: `{"trigger":"api","parameters":{"force":"true"}}` +4. Wait for the map run to finish with `status=Succeeded` via `GET /jobs/{runId}`. +5. Recheck `GET /api/v1/advisory-sources/kev/freshness` and confirm the KEV rollups are repopulated: + - `sourceDocumentCount > 0` + - `canonicalAdvisoryCount > 0` + - `cveCount > 0` + - `totalAdvisories > 0` +6. Verify PostgreSQL recovery state: + - `vuln.advisories` rows with advisory keys prefixed `kev/` have non-null `source_id` + - `vuln.advisory_source_edge` contains rows for the KEV source + +This path reuses the most recent cached KEV catalog and does not wait for CISA to publish a newer document. It is the required operator recovery step after shipping KEV mapper or persistence fixes onto a host that already cached the current catalog. + +### 2.4 Production Monitoring - Alert when `rate(kev_fetch_success_total[8h]) == 0` during working hours (daily cadence breach) and when `increase(kev_fetch_failures_total[1h]) > 0`. - Page the on-call if `increase(kev_parse_failures_total{reason="schema"}[6h]) > 0`—this usually signals an upstream payload change. Treat repeated `reason="download"` spikes as networking issues to the mirror. diff --git a/docs/modules/concelier/operations/connectors/ghsa.md b/docs/modules/concelier/operations/connectors/ghsa.md index bc5c8177f..df4bc6290 100644 --- a/docs/modules/concelier/operations/connectors/ghsa.md +++ b/docs/modules/concelier/operations/connectors/ghsa.md @@ -1,123 +1,103 @@ -# Concelier GHSA Connector – Operations Runbook - -_Last updated: 2025-10-16_ - -## 1. Overview -The GitHub Security Advisories (GHSA) connector pulls advisory metadata from the GitHub REST API `/security/advisories` endpoint. GitHub enforces both primary and secondary rate limits, so operators must monitor usage and configure retries to avoid throttling incidents. - -## 2. Rate-limit telemetry -The connector now surfaces rate-limit headers on every fetch and exposes the following metrics via OpenTelemetry: - -| Metric | Description | Tags | -|--------|-------------|------| -| `ghsa.ratelimit.limit` (histogram) | Samples the reported request quota at fetch time. | `phase` = `list` or `detail`, `resource` (e.g., `core`). | -| `ghsa.ratelimit.remaining` (histogram) | Remaining requests returned by `X-RateLimit-Remaining`. | `phase`, `resource`. | -| `ghsa.ratelimit.reset_seconds` (histogram) | Seconds until `X-RateLimit-Reset`. | `phase`, `resource`. | -| `ghsa.ratelimit.headroom_pct` (histogram) | Percentage of the quota still available (`remaining / limit * 100`). | `phase`, `resource`. | -| `ghsa.ratelimit.headroom_pct_current` (observable gauge) | Latest headroom percentage reported per resource. | `phase`, `resource`. | -| `ghsa.ratelimit.exhausted` (counter) | Incremented whenever GitHub returns a zero remaining quota and the connector delays before retrying. | `phase`. | - -### Dashboards & alerts -- Plot `ghsa.ratelimit.remaining` as the latest value to watch the runway. Alert when the value stays below **`RateLimitWarningThreshold`** (default `500`) for more than 5 minutes. -- Use `ghsa.ratelimit.headroom_pct_current` to visualise remaining quota % — paging once it sits below **10 %** for longer than a single reset window helps avoid secondary limits. -- Raise a separate alert on `increase(ghsa.ratelimit.exhausted[15m]) > 0` to catch hard throttles. -- Overlay `ghsa.fetch.attempts` vs `ghsa.fetch.failures` to confirm retries are effective. - -## 3. Logging signals -When `X-RateLimit-Remaining` falls below `RateLimitWarningThreshold`, the connector emits: -``` -GHSA rate limit warning: remaining {Remaining}/{Limit} for {Phase} {Resource} (headroom {Headroom}%) -``` -When GitHub reports zero remaining calls, the connector logs and sleeps for the reported `Retry-After`/`X-RateLimit-Reset` interval (falling back to `SecondaryRateLimitBackoff`). - -After the quota recovers above the warning threshold the connector writes an informational log with the refreshed remaining/headroom, letting operators clear alerts quickly. - -## 4. Configuration knobs (`concelier.yaml`) -```yaml -concelier: - sources: - ghsa: - apiToken: "${GITHUB_PAT}" - pageSize: 50 - requestDelay: "00:00:00.200" - failureBackoff: "00:05:00" - rateLimitWarningThreshold: 500 # warn below this many remaining calls - secondaryRateLimitBackoff: "00:02:00" # fallback delay when GitHub omits Retry-After -``` - -### Recommendations -- Increase `requestDelay` in air-gapped or burst-heavy deployments to smooth token consumption. -- Lower `rateLimitWarningThreshold` only if your dashboards already page on the new histogram; never set it negative. -- For bots using a low-privilege PAT, keep `secondaryRateLimitBackoff` at ≥60 seconds to respect GitHub’s secondary-limit guidance. - -#### Default job schedule - -| Job kind | Cron | Timeout | Lease | -|----------|------|---------|-------| -| `source:ghsa:fetch` | `1,11,21,31,41,51 * * * *` | 6 minutes | 4 minutes | -| `source:ghsa:parse` | `3,13,23,33,43,53 * * * *` | 5 minutes | 4 minutes | -| `source:ghsa:map` | `5,15,25,35,45,55 * * * *` | 5 minutes | 4 minutes | - -These defaults spread GHSA stages across the hour so fetch completes before parse/map fire. Override them via `concelier.jobs.definitions[...]` when coordinating multiple connectors on the same runner. - -## 5. Provisioning credentials - -Concelier requires a GitHub personal access token (classic) with the **`read:org`** and **`security_events`** scopes to pull GHSA data. Store it as a secret and reference it via `concelier.sources.ghsa.apiToken`. - -### Docker Compose (stack operators) -```yaml -services: - concelier: - environment: - CONCELIER__SOURCES__GHSA__APITOKEN: /run/secrets/ghsa_pat - secrets: - - ghsa_pat - -secrets: - ghsa_pat: - file: ./secrets/ghsa_pat.txt # contains only the PAT value -``` - -### Helm values (cluster operators) -```yaml -concelier: - extraEnv: - - name: CONCELIER__SOURCES__GHSA__APITOKEN - valueFrom: - secretKeyRef: - name: concelier-ghsa - key: apiToken - -extraSecrets: - concelier-ghsa: - apiToken: "" -``` - -After rotating the PAT, restart the Concelier workers (or run `kubectl rollout restart deployment/concelier`) to ensure the configuration reloads. - -When enabling GHSA the first time, run a staged backfill: - -1. Trigger `source:ghsa:fetch` manually (CLI or API) outside of peak hours. -2. Watch `concelier.jobs.health` for the GHSA jobs until they report `healthy`. -3. Allow the scheduled cron cadence to resume once the initial backlog drains (typically < 30 minutes). - -## 6. Runbook steps when throttled -1. Check `ghsa.ratelimit.exhausted` for the affected phase (`list` vs `detail`). -2. Confirm the connector is delaying—logs will show `GHSA rate limit exhausted...` with the chosen backoff. -3. If rate limits stay exhausted: - - Verify no other jobs are sharing the PAT. - - Temporarily reduce `MaxPagesPerFetch` or `PageSize` to shrink burst size. - - Consider provisioning a dedicated PAT (GHSA permissions only) for Concelier. -4. After the quota resets, reset `rateLimitWarningThreshold`/`requestDelay` to their normal values and monitor the histograms for at least one hour. - -## 7. Alert integration quick reference -- Prometheus: `ghsa_ratelimit_remaining_bucket` (from histogram) – use `histogram_quantile(0.99, ...)` to trend capacity. -- VictoriaMetrics: `LAST_over_time(ghsa_ratelimit_remaining_sum[5m])` for simple last-value graphs. -- Grafana: stack remaining + used to visualise total limit per resource. - -## 8. Canonical metric fallback analytics -When GitHub omits CVSS vectors/scores, the connector now assigns a deterministic canonical metric id in the form `ghsa:severity/` and publishes it to Merge so severity precedence still resolves against GHSA even without CVSS data. - -- Metric: `ghsa.map.canonical_metric_fallbacks` (counter) with tags `severity`, `canonical_metric_id`, `reason=no_cvss`. -- Monitor the counter alongside Merge parity checks; a sudden spike suggests GitHub is shipping advisories without vectors and warrants cross-checking downstream exporters. -- Because the canonical id feeds Merge, parity dashboards should overlay this metric to confirm fallback advisories continue to merge ahead of downstream sources when GHSA supplies more recent data. +# Concelier GHSA Connector - Operations Runbook + +_Last updated: 2026-04-22_ + +## 1. Overview + +The GitHub Security Advisories (GHSA) connector pulls advisory metadata from the GitHub REST API `/security/advisories` endpoint. GitHub enforces both primary and secondary rate limits, so operators must monitor usage and configure retries to avoid throttling incidents. + +## 2. Rate-limit telemetry + +The connector surfaces rate-limit headers on every fetch and exposes the following metrics via OpenTelemetry: + +| Metric | Description | Tags | +| --- | --- | --- | +| `ghsa.ratelimit.limit` | Samples the reported request quota at fetch time. | `phase`, `resource` | +| `ghsa.ratelimit.remaining` | Remaining requests returned by `X-RateLimit-Remaining`. | `phase`, `resource` | +| `ghsa.ratelimit.reset_seconds` | Seconds until `X-RateLimit-Reset`. | `phase`, `resource` | +| `ghsa.ratelimit.headroom_pct` | Percentage of quota still available. | `phase`, `resource` | +| `ghsa.ratelimit.headroom_pct_current` | Latest headroom percentage per resource. | `phase`, `resource` | +| `ghsa.ratelimit.exhausted` | Incremented whenever GitHub returns zero remaining quota and the connector delays before retrying. | `phase` | + +Dashboards and alerts: + +- Alert when `ghsa.ratelimit.remaining` stays below `RateLimitWarningThreshold` for more than 5 minutes. +- Page if `ghsa.ratelimit.headroom_pct_current` remains below 10% for longer than a reset window. +- Alert on `increase(ghsa.ratelimit.exhausted[15m]) > 0`. + +## 3. Configuration paths + +Primary operator path: + +- Web UI: **Security Posture -> Configure Sources** or **Ops -> Operations -> Feeds & Airgap -> Configure Sources** +- CLI: + ```bash + stella db connectors configure ghsa \ + --server https://concelier.example.internal \ + --set apiToken=github_pat_xxx + ``` + +The persisted source configuration key is `apiToken`. Secrets are retained server-side and are not echoed back after storage. + +Compatibility fallback (`concelier.yaml`): + +```yaml +concelier: + sources: + ghsa: + apiToken: "${GITHUB_PAT}" + pageSize: 50 + requestDelay: "00:00:00.200" + failureBackoff: "00:05:00" + rateLimitWarningThreshold: 500 + secondaryRateLimitBackoff: "00:02:00" +``` + +Recommendations: + +- Increase `requestDelay` in burst-heavy or mirrored deployments. +- Lower `rateLimitWarningThreshold` only if dashboards already page on the telemetry above. +- Keep `secondaryRateLimitBackoff` at 60 seconds or more for low-privilege tokens. + +## 4. Credential acquisition + +Where to sign in: + +- `https://github.com/settings/personal-access-tokens` + +What to create: + +- A GitHub personal access token or a GitHub App token that can call the security advisories REST endpoints + +Upstream notes: + +- GitHub documents that the global security advisories endpoint can serve public reviewed advisories anonymously and works with fine-grained PATs. +- The current StellaOps GHSA connector still validates that `apiToken` is present, so skipping the token is not supported for this connector path. +- If the target organization uses SAML SSO, authorize the PAT for that org after creation. +- Some organizations restrict classic PATs or require PAT approval. That policy is external to StellaOps. + +Legacy host/env distribution remains available for older deployments, but it is no longer the preferred operator path. + +## 5. Default job schedule + +| Job kind | Cron | Timeout | Lease | +| --- | --- | --- | --- | +| `source:ghsa:fetch` | `1,11,21,31,41,51 * * * *` | 6 minutes | 4 minutes | +| `source:ghsa:parse` | `3,13,23,33,43,53 * * * *` | 5 minutes | 4 minutes | +| `source:ghsa:map` | `5,15,25,35,45,55 * * * *` | 5 minutes | 4 minutes | + +These defaults spread GHSA stages across the hour so fetch completes before parse/map fire. + +## 6. Runbook steps when throttled + +1. Check `ghsa.ratelimit.exhausted` for the affected phase. +2. Confirm the connector is delaying before retry. +3. If exhaustion persists: + - Verify no unrelated jobs are sharing the token. + - Temporarily reduce `MaxPagesPerFetch` or `PageSize`. + - Consider provisioning a dedicated token. +4. After quota resets, return settings to their normal values and monitor for at least one hour. + +## 7. Canonical metric fallback analytics + +When GitHub omits CVSS vectors or scores, the connector assigns a deterministic canonical metric ID in the form `ghsa:severity/` and publishes it to Merge so severity precedence still resolves against GHSA even without CVSS data. diff --git a/docs/modules/concelier/operations/connectors/jvn.md b/docs/modules/concelier/operations/connectors/jvn.md index 1d378c8ca..4f5a68d22 100644 --- a/docs/modules/concelier/operations/connectors/jvn.md +++ b/docs/modules/concelier/operations/connectors/jvn.md @@ -3,7 +3,7 @@ _Last updated: 2026-01-16_ ## 1. Overview -The JVN connector ingests Japan Vulnerability Notes (JVN) advisories and maps them to canonical IDs. +The JVN connector ingests Japan Vulnerability Notes (JVN) advisories and maps them to canonical IDs. The runtime/catalog source ID is `jpcert`. ## 2. Authentication - No authentication required for public feeds. @@ -12,13 +12,15 @@ The JVN connector ingests Japan Vulnerability Notes (JVN) advisories and maps th ```yaml concelier: sources: - jvn: + jpcert: baseUri: "" maxDocumentsPerFetch: 20 fetchTimeout: "00:00:45" requestDelay: "00:00:00" ``` +Legacy `concelier:sources:jvn` binding is still accepted for compatibility, but operators should use `concelier:sources:jpcert` for new installs and UI-driven setup. + ## 4. Offline and air-gapped deployments - Mirror JVN feeds into the Offline Kit and repoint `baseUri` to the mirror. diff --git a/docs/modules/concelier/operations/connectors/kisa.md b/docs/modules/concelier/operations/connectors/kisa.md index 77b425262..b3da681a3 100644 --- a/docs/modules/concelier/operations/connectors/kisa.md +++ b/docs/modules/concelier/operations/connectors/kisa.md @@ -1,19 +1,19 @@ # Concelier KISA Connector Operations -Operational guidance for the Korea Internet & Security Agency (KISA / KNVD) connector (`source:kisa:*`). Pair this with the engineering brief in `docs/dev/kisa_connector_notes.md`. +Operational guidance for the Korea Internet & Security Agency (KISA / KNVD) connector. The runtime/catalog source ID is `krcert`, so the live fetch pipeline runs as `source:krcert:*`. Pair this with the engineering brief in `docs/dev/kisa_connector_notes.md`. ## 1. Prerequisites - Outbound HTTPS (or mirrored cache) for `https://knvd.krcert.or.kr/`. -- Connector options defined under `concelier:sources:kisa`: +- Connector options defined under `concelier:sources:krcert`: ```yaml concelier: sources: - kisa: - feedUri: "https://knvd.krcert.or.kr/rss/securityInfo.do" - detailApiUri: "https://knvd.krcert.or.kr/rssDetailData.do" - detailPageUri: "https://knvd.krcert.or.kr/detailDos.do" + krcert: + feedUri: "https://knvd.krcert.or.kr/rss/securityInfo.do" + detailApiUri: "https://knvd.krcert.or.kr/rssDetailData.do" + detailPageUri: "https://knvd.krcert.or.kr/detailDos.do" maxAdvisoriesPerFetch: 10 requestDelay: "00:00:01" failureBackoff: "00:05:00" @@ -21,12 +21,14 @@ concelier: > Ensure the URIs stay absolute—Concelier adds the `feedUri`/`detailApiUri` hosts to the HttpClient allow-list automatically. -## 2. Staging Smoke Test +Legacy `concelier:sources:kisa` binding is still accepted for compatibility, but new setups should use `concelier:sources:krcert`. + +## 2. Staging Smoke Test 1. Restart the Concelier workers so the KISA options bind. 2. Run a full connector cycle: - - CLI: run `stella db fetch --source kisa --stage fetch`, then `--stage parse`, then `--stage map`. - - REST: `POST /jobs/run { "kind": "source:kisa:fetch", "chain": ["source:kisa:parse", "source:kisa:map"] }` + - CLI: run `stella db fetch --source krcert --stage fetch`, then `--stage parse`, then `--stage map`. + - REST: `POST /jobs/run { "kind": "source:krcert:fetch", "chain": ["source:krcert:parse", "source:krcert:map"] }` 3. Confirm telemetry (Meter `StellaOps.Concelier.Connector.Kisa`): - `kisa.feed.success`, `kisa.feed.items` - `kisa.detail.success` / `.failures` @@ -42,7 +44,9 @@ concelier: - `raw_documents` table metadata has `kisa.idx`, `kisa.category`, `kisa.title`. - `dtos` table contains `schemaVersion="kisa.detail.v1"`. - `advisories` table includes aliases (`IDX`, CVE) and `language="ko"`. - - `source_states` entry for `kisa` shows recent `cursor.lastFetchAt`. + - `source_states` entry for `krcert` shows recent `cursor.lastFetchAt`. + +Telemetry and fixture naming still use the historical `kisa.*` metric namespace, so dashboards should keep those series names even though the runtime source ID is now `krcert`. ## 3. Production Monitoring diff --git a/docs/modules/concelier/operations/connectors/msrc.md b/docs/modules/concelier/operations/connectors/msrc.md index 097ed7fbe..d578d3493 100644 --- a/docs/modules/concelier/operations/connectors/msrc.md +++ b/docs/modules/concelier/operations/connectors/msrc.md @@ -1,86 +1,68 @@ -# Concelier MSRC Connector – Azure AD Onboarding Brief - -_Drafted: 2025-10-15_ - -## 1. App registration requirements - -- **Tenant**: shared StellaOps production Azure AD. -- **Application type**: confidential client (web/API) issuing client credentials. -- **API permissions**: `api://api.msrc.microsoft.com/.default` (Application). Admin consent required once. -- **Token audience**: `https://api.msrc.microsoft.com/`. -- **Grant type**: client credentials. Concelier will request tokens via `POST https://login.microsoftonline.com/{tenantId}/oauth2/v2.0/token`. - -## 2. Secret/credential policy - -- Maintain two client secrets (primary + standby) rotating every 90 days. -- Store secrets in the Concelier secrets vault; Offline Kit deployments must mirror the secret payloads in their encrypted store. -- Record rotation cadence in Ops runbook and update Concelier configuration (`CONCELIER__SOURCES__VNDR__MSRC__CLIENTSECRET`) ahead of expiry. - -## 3. Concelier configuration sample - -```yaml -concelier: - sources: - vndr.msrc: - tenantId: "" - clientId: "" - clientSecret: "" - apiVersion: "2024-08-01" - locale: "en-US" - requestDelay: "00:00:00.250" - failureBackoff: "00:05:00" - cursorOverlapMinutes: 10 - downloadCvrf: false # set true to persist CVRF ZIP alongside JSON detail -``` - -## 4. CVRF artefacts - -- The MSRC REST payload exposes `cvrfUrl` per advisory. Current connector persists the link as advisory metadata and reference; it does **not** download the ZIP by default. -- Ops should mirror CVRF ZIPs when preparing Offline Kits so air-gapped deployments can reconcile advisories without direct internet access. -- Once Offline Kit storage guidelines are finalised, extend the connector configuration with `downloadCvrf: true` to enable automatic attachment retrieval. - -### 4.1 State seeding helper - -Use `src/Tools/SourceStateSeeder` to queue historical advisories (detail JSON + optional CVRF artefacts) for replay without manual database edits. Example seed file: - -```json -{ - "source": "vndr.msrc", - "cursor": { - "lastModifiedCursor": "2024-01-01T00:00:00Z" - }, - "documents": [ - { - "uri": "https://api.msrc.microsoft.com/sug/v2.0/vulnerability/ADV2024-0001", - "contentFile": "./seeds/adv2024-0001.json", - "contentType": "application/json", - "metadata": { "msrc.vulnerabilityId": "ADV2024-0001" }, - "addToPendingDocuments": true - }, - { - "uri": "https://download.microsoft.com/msrc/2024/ADV2024-0001.cvrf.zip", - "contentFile": "./seeds/adv2024-0001.cvrf.zip", - "contentType": "application/zip", - "status": "mapped", - "addToPendingDocuments": false - } - ] -} -``` - -Run the helper: - -```bash -dotnet run --project src/Tools/SourceStateSeeder -- \ - --connection-string "Host=localhost;Database=stellaops;Username=stella;Password=..." \ - --schema vuln \ - --input seeds/msrc-backfill.json -``` - -Any documents marked `addToPendingDocuments` will appear in the connector cursor; `DownloadCvrf` can remain disabled if the ZIP artefact is pre-seeded. - -## 5. Outstanding items - -- Ops to confirm tenant/app names and provide client credentials through the secure channel. -- Connector team monitors token cache health (already implemented); validate instrumentation once Ops supplies credentials. -- Offline Kit packaging: add encrypted blob containing client credentials with rotation instructions. +# Concelier MSRC Connector - Microsoft Entra Onboarding Brief + +_Last updated: 2026-04-22_ + +## 1. App registration requirements + +Sign in at `https://entra.microsoft.com` and open **App registrations**. + +Create or select a confidential client application that will use client credentials. Capture: + +- `tenantId` = Directory (tenant) ID +- `clientId` = Application (client) ID +- `clientSecret` = client secret value from **Certificates & secrets** + +Primary operator path: + +- Web UI: **Security Posture -> Configure Sources** or **Ops -> Operations -> Feeds & Airgap -> Configure Sources** +- CLI: + ```bash + stella db connectors configure microsoft \ + --server https://concelier.example.internal \ + --set tenantId=... \ + --set clientId=... \ + --set clientSecret=... + ``` + +Notes: + +- Microsoft recommends certificates over client secrets for production applications. The current StellaOps MSRC operator entry path is client-secret based. +- MSRC is not documented here as a separate paid product, but you do need a Microsoft Entra tenant plus permission to register the app and grant the required consent. + +## 2. Secret and credential policy + +- Maintain two client secrets rotating every 90 days. +- Prefer storing the active secret through the StellaOps source configuration UI or CLI. +- Offline Kit deployments that still mirror secrets out of band must mirror the secret payloads in their encrypted store. +- Legacy host configuration remains a compatibility fallback only. + +## 3. Compatibility fallback (`concelier.yaml`) + +```yaml +concelier: + sources: + microsoft: + tenantId: "" + clientId: "" + clientSecret: "" + apiVersion: "2024-08-01" + locale: "en-US" + requestDelay: "00:00:00.250" + failureBackoff: "00:05:00" + cursorOverlapMinutes: 10 + downloadCvrf: false +``` + +The runtime source ID is `microsoft`, and the connector still binds the legacy `vndr:msrc` section for compatibility. + +## 4. CVRF artefacts + +- The MSRC REST payload exposes `cvrfUrl` per advisory. +- Current connector behavior records the link as advisory metadata and reference; it does not download the ZIP by default. +- Mirror CVRF ZIPs when preparing Offline Kits if air-gapped deployments need them. + +## 5. Outstanding items + +- Ops must confirm the tenant and app used for production MSRC access. +- Connector owners should validate token cache health once credentials are supplied. +- If certificate-based auth is later required, StellaOps needs a dedicated certificate entry path before switching the runbook away from client secrets. diff --git a/docs/modules/concelier/operations/connectors/oracle.md b/docs/modules/concelier/operations/connectors/oracle.md index ded3d0066..906fb159d 100644 --- a/docs/modules/concelier/operations/connectors/oracle.md +++ b/docs/modules/concelier/operations/connectors/oracle.md @@ -1,26 +1,75 @@ # Concelier Oracle CPU Connector - Operations Runbook -_Last updated: 2026-01-16_ +_Last updated: 2026-04-22_ ## 1. Overview -The Oracle connector ingests Oracle Critical Patch Update advisories and maps them to canonical IDs. + +The Concelier Oracle connector ingests Oracle Critical Patch Update advisories and maps them to canonical IDs. + +For the default Excititor VEX mirror bootstrap, Stella Ops discovers Oracle updates from Oracle public security RSS and derives the matching CSAF JSON document URIs deterministically. ## 2. Authentication + - No authentication required for public advisories. -## 3. Configuration (`concelier.yaml`) +## 3. Configuration paths + +Primary operator path: + +- Web UI: **Security Posture -> Configure Sources** or **Ops -> Operations -> Feeds & Airgap -> Configure Sources** +- CLI: + ```bash + stella db connectors configure oracle \ + --server https://concelier.example.internal \ + --set calendarUris=https://www.oracle.com/security-alerts/ + ``` + +Use the UI/CLI path only when overriding the default Oracle landing page or pinning specific advisories for mirrored or offline flows. Leaving the URI fields blank keeps the built-in public default. + +Compatibility fallback (`concelier.yaml`): + ```yaml concelier: sources: - oracle: - baseUri: "" - maxDocumentsPerFetch: 20 - fetchTimeout: "00:00:45" - requestDelay: "00:00:00" + vndr: + oracle: + calendarUris: + - "https://www.oracle.com/security-alerts/" + advisoryUris: [] + requestDelay: "00:00:01" ``` -## 4. Offline and air-gapped deployments -- Mirror CPU advisories into the Offline Kit and repoint `baseUri` to the mirror. +Notes: -## 5. Common failure modes -- Schedule drift during quarterly CPU updates. +- Concelier uses Oracle calendar or index pages for the advisory connector. +- `calendarUris` and `advisoryUris` are both valid. +- At least one of them must be configured unless you rely on the built-in default Oracle landing page. +- `calendarUris` should point at Oracle calendar or index pages; the connector filters discovered links down to concrete `/security-alerts/*.html` advisory pages. + +Example: pinning a specific quarterly CPU advisory directly + +```yaml +concelier: + sources: + vndr: + oracle: + advisoryUris: + - "https://www.oracle.com/security-alerts/cpuapr2025.html" + requestDelay: "00:00:00" +``` + +## 4. Excititor default public VEX bootstrap + +- Discovery RSS: `https://www.oracle.com/ocom/groups/public/@otn/documents/webcontent/rss-otn-sec.xml` +- Derived CSAF base: `https://www.oracle.com/docs/tech/security-alerts/` + +## 5. Offline and air-gapped deployments + +- Mirror CPU advisories into the Offline Kit and repoint `calendarUris` or `advisoryUris` to the mirrored HTML endpoints for advisory ingestion. +- For Excititor mirror bootstrap, mirror the Oracle RSS feed and the derived `/docs/tech/security-alerts/*csaf.json` documents. + +## 6. Common failure modes + +- Schedule drift during quarterly CPU updates +- Oracle renaming RSS items or advisory stems in a way that changes the derived CSAF JSON path +- Mirroring only the HTML pages while omitting the derived CSAF JSON payloads needed by Excititor diff --git a/docs/modules/concelier/operations/connectors/ubuntu.md b/docs/modules/concelier/operations/connectors/ubuntu.md index 37289e239..4f07c9544 100644 --- a/docs/modules/concelier/operations/connectors/ubuntu.md +++ b/docs/modules/concelier/operations/connectors/ubuntu.md @@ -1,9 +1,11 @@ # Concelier Ubuntu USN Connector - Operations Runbook -_Last updated: 2026-01-16_ +_Last updated: 2026-04-21_ ## 1. Overview -The Ubuntu connector ingests Ubuntu Security Notices (USN) and maps advisories to Ubuntu package versions. +The Concelier Ubuntu connector ingests Ubuntu Security Notices (USN) and maps advisories to Ubuntu package versions. + +The same public notice feed also backs the default Excititor VEX mirror bootstrap. Ubuntu does not currently publish native CSAF for this path, so Excititor synthesizes deterministic CSAF documents from the notice JSON while preserving the upstream source URI in metadata. ## 2. Authentication - No authentication required for public feeds. @@ -19,8 +21,23 @@ concelier: requestDelay: "00:00:00" ``` -## 4. Offline and air-gapped deployments -- Mirror USN feeds into the Offline Kit and repoint `baseUri` to the mirror. +## 4. Excititor default public VEX bootstrap +- Index URI: `https://ubuntu.com/security/notices.json` +- Notice detail base URI: `https://ubuntu.com/security/notices/` +- Default page size: `20` +- Default max notices per fetch: `60` +- Default resume overlap: `3.00:00:00` -## 5. Common failure modes +Operational guidance: +- Keep the small page size and bounded fetch count unless Canonical publishes a stronger bulk-ingest contract. This avoids burst-fetching the full notice history during mirror bootstrap. +- Keep the resume overlap enabled so the mirror rechecks recently updated notices without needing a full backfill. +- Mirror both the paged `notices.json` index responses and the per-notice `USN-xxxx-x.json` documents for offline kits. + +## 5. Offline and air-gapped deployments +- Mirror USN feeds into the Offline Kit and repoint `baseUri` to the mirror for advisory ingestion. +- For Excititor mirror bootstrap, mirror the `notices.json` index plus the per-notice JSON documents under the same path layout so synthesized CSAF documents remain deterministic. + +## 6. Common failure modes - USN schema updates or missing release references. +- Per-notice JSON documents lagging behind the index update window. +- Overly aggressive page sizes or fetch counts causing avoidable upstream pressure during first-run bootstrap. diff --git a/docs/modules/concelier/operations/source-credentials.md b/docs/modules/concelier/operations/source-credentials.md new file mode 100644 index 000000000..76f25d4f8 --- /dev/null +++ b/docs/modules/concelier/operations/source-credentials.md @@ -0,0 +1,135 @@ +# Advisory Source Credential Entry + +_Last updated: 2026-04-22_ + +## 1. Purpose + +Stella Ops now supports operator-supplied advisory source settings through the product surfaces that operators already use: + +- Web UI source management +- `stella db connectors configure ...` in the CLI + +Environment variables and host-local `concelier.yaml` values remain compatibility fallbacks for older deployments, but the primary operator path for supported advisory sources is now persisted source configuration owned by Concelier itself. + +## 2. Operator entry paths + +### Web UI + +Use either of these routes: + +- **Security Posture -> Configure Sources** +- **Ops -> Operations -> Feeds & Airgap -> Configure Sources** + +Then: + +1. Expand the source card. +2. Open **Stored Connector Configuration**. +3. Enter or update the source fields. +4. Save the configuration. + +Sensitive fields never round-trip back to the browser. A stored secret is shown only as retained state. Leaving a password field blank keeps the retained secret. Explicitly checking the clear control removes the stored secret. + +### CLI + +Inspect current persisted source configuration: + +```bash +stella db connectors configure ghsa --server https://concelier.example.internal +stella db connectors configure cisco --server https://concelier.example.internal +``` + +Update a source: + +```bash +stella db connectors configure ghsa \ + --server https://concelier.example.internal \ + --set apiToken=github_pat_xxx + +stella db connectors configure cisco \ + --server https://concelier.example.internal \ + --set clientId=... \ + --set clientSecret=... + +stella db connectors configure microsoft \ + --server https://concelier.example.internal \ + --set tenantId=... \ + --set clientId=... \ + --set clientSecret=... + +stella db connectors configure oracle \ + --server https://concelier.example.internal \ + --set calendarUris=https://www.oracle.com/security-alerts/,https://mirror.example.internal/oracle/ + +stella db connectors configure adobe \ + --server https://concelier.example.internal \ + --set indexUri=https://mirror.example.internal/adobe/security-bulletin.html \ + --set additionalIndexUris=https://mirror.example.internal/adobe/archive-1.html;https://mirror.example.internal/adobe/archive-2.html + +stella db connectors configure chromium \ + --server https://concelier.example.internal \ + --set feedUri=https://mirror.example.internal/chromium/atom.xml +``` + +Clear stored fields: + +```bash +stella db connectors configure ghsa \ + --server https://concelier.example.internal \ + --clear apiToken +``` + +Notes: + +- `--set` accepts `key=value`. +- Multi-value URI fields such as `calendarUris`, `advisoryUris`, and `additionalIndexUris` accept comma-, semicolon-, or newline-separated absolute URIs. +- The current CLI path places literal values on the command line. If shell-history exposure is unacceptable for a secret, prefer the Web UI path or use an operator-approved secure shell/history procedure. + +## 3. Credential acquisition matrix + +| Source | Where to sign in or look | What to create or capture | Can the config be skipped? | Entitlement / paywall notes | +| --- | --- | --- | --- | --- | +| `ghsa` | `https://github.com/settings/personal-access-tokens` or a GitHub App owned by your org | `apiToken` | Not for the current StellaOps GHSA connector path. The upstream API can expose public reviewed advisories anonymously, but the current StellaOps connector still expects a token. | No separate GHSA paywall. GitHub org PAT policy or SAML SSO may require approval or token authorization. | +| `cisco` | `https://apiconsole.cisco.com` -> **My Apps & Keys** | `clientId`, `clientSecret` for a Service / Client Credentials app bound to Cisco PSIRT openVuln API | Not for the authenticated Concelier Cisco advisory connector. | No separate StellaOps-side fee, but a Cisco.com account, terms acceptance, and visible openVuln entitlement are required. This is separate from the public Cisco CSAF VEX bootstrap, which stays credential-free. | +| `microsoft` | `https://entra.microsoft.com` -> **App registrations** | `tenantId`, `clientId`, `clientSecret` for a confidential client allowed to use MSRC client credentials | Not for the MSRC advisory connector. | No separate documented MSRC paywall, but you need a Microsoft Entra tenant plus permission to register the app and grant the required consent. | +| `oracle` | Public Oracle security pages | Usually nothing. Optionally capture mirrored `calendarUris` or pinned `advisoryUris`. | Yes, if the default Oracle security alerts landing page is acceptable. Configure it only when pinning or mirroring. | Public, no login or paywall required for the default path. | +| `adobe` | Public Adobe bulletin index | Usually nothing. Optionally capture a mirrored `indexUri` and `additionalIndexUris`. | Yes, for the default public Adobe index. Configure it only when overriding or mirroring the public endpoints. | Public, no login or paywall required for the default path. | +| `chromium` | Public Chrome Releases Atom feed | Usually nothing. Optionally capture a mirrored `feedUri`. | Yes, for the default public Chromium feed. Configure it only when overriding or mirroring the public endpoint. | Public, no login or paywall required for the default path. | + +## 4. What operators should actually look for + +### GHSA + +- Personal access token page or org-owned GitHub App credentials +- If the organization enforces SAML SSO or PAT approval, make sure the token is authorized for the target org after creation +- The StellaOps field name is `apiToken` + +### Cisco + +- Cisco API Console entry for **Cisco PSIRT openVuln API** +- Application type: `Service` +- Grant type: `Client Credentials` +- Capture the generated `clientId` and `clientSecret` + +### Microsoft / MSRC + +- Microsoft Entra **Application (client) ID** +- Microsoft Entra **Directory (tenant) ID** +- A newly created **Client secret** value +- Confirm admin consent and the app permissions expected by your MSRC onboarding process before storing the values in StellaOps + +### Oracle / Adobe / Chromium + +- No credential creation is required +- Only collect alternate URIs if you are pointing Concelier at an approved internal mirror or pinning a specific public advisory page + +## 5. References + +- GitHub PAT management: +- GitHub global security advisories REST API: +- GitHub SSO authorization for PATs: +- Cisco PSIRT openVuln authentication: +- Microsoft Entra app registration quickstart: +- Microsoft Entra application credentials: +- Adobe bulletin index: +- Oracle security alerts: +- Chrome Releases Atom feed: diff --git a/docs/modules/concelier/source-coverage.md b/docs/modules/concelier/source-coverage.md index fcea9c71a..172a7470f 100644 --- a/docs/modules/concelier/source-coverage.md +++ b/docs/modules/concelier/source-coverage.md @@ -7,9 +7,9 @@ Last updated: 2026-04-06 | Metric | Count | |--------|-------| | Total sources defined | 70 | -| Connectors implemented | 33 | -| Coverage rate | 47% | -| Missing connectors | 37 | +| Connectors implemented | 34 | +| Coverage rate | 49% | +| Missing connectors | 36 | ## Coverage by Category @@ -79,7 +79,7 @@ Ecosystem advisories are currently routed through OSV/GHSA. Direct connectors wo | azure | Azure Security Advisories | P3 | Missing | | gcp | GCP Security Bulletins | P3 | Missing | -### National CERTs (7/13 — 54%) +### National CERTs (9/14 — 64%) | Source | Display Name | Connector | Status | |--------|-------------|-----------|--------| @@ -87,6 +87,7 @@ Ecosystem advisories are currently routed through OSV/GHSA. Direct connectors wo | cert-fr | CERT-FR (France) | `Connector.CertFr` | Complete | | cert-de | CERT-Bund (Germany) | `Connector.CertBund` | Complete | | jpcert | JPCERT/CC (Japan) | `Connector.Jvn` | Complete | +| auscert | AusCERT (Australia) | `Connector.Acsc` | Complete | | krcert | KrCERT (South Korea) | `Connector.Kisa` | Complete | | cert-in | CERT-In (India) | `Connector.CertIn` | Complete | | fstec-bdu | FSTEC BDU (Russia) | `Connector.RuBdu` | Complete | @@ -142,7 +143,6 @@ Ecosystem advisories are currently routed through OSV/GHSA. Direct connectors wo | pypa | PyPA Advisory DB | — | Missing (P3) | | govuln | Go Vuln DB | — | Missing (P3) | | bundler-audit | Ruby Advisory DB | — | Missing (P3) | -| auscert | AusCERT (Australia) | — | Missing (P4) | | cert-pl | CERT.PL (Poland) | — | Missing (P4) | --- diff --git a/src/Concelier/StellaOps.Concelier.WebService/Extensions/AdvisorySourceEndpointExtensions.cs b/src/Concelier/StellaOps.Concelier.WebService/Extensions/AdvisorySourceEndpointExtensions.cs index ac4c1afcd..7ea83d733 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Extensions/AdvisorySourceEndpointExtensions.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Extensions/AdvisorySourceEndpointExtensions.cs @@ -2,6 +2,7 @@ using HttpResults = Microsoft.AspNetCore.Http.Results; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Caching.Memory; using StellaOps.Auth.ServerIntegration.Tenancy; +using StellaOps.Concelier.Core.Sources; using StellaOps.Concelier.Persistence.Postgres.Repositories; namespace StellaOps.Concelier.WebService.Extensions; @@ -36,7 +37,7 @@ internal static class AdvisorySourceEndpointExtensions } var records = await readRepository.ListAsync(includeDisabledValue, cancellationToken).ConfigureAwait(false); - var items = records.Select(MapListItem).ToList(); + var items = BuildListItems(records, includeDisabledValue); var response = new AdvisorySourceListResponse { @@ -68,15 +69,18 @@ internal static class AdvisorySourceEndpointExtensions } var records = await readRepository.ListAsync(includeDisabled: true, cancellationToken).ConfigureAwait(false); + var items = BuildListItems(records, includeDisabled: true); var response = new AdvisorySourceSummaryResponse { - TotalSources = records.Count, - HealthySources = records.Count(r => string.Equals(r.FreshnessStatus, "healthy", StringComparison.OrdinalIgnoreCase)), - WarningSources = records.Count(r => string.Equals(r.FreshnessStatus, "warning", StringComparison.OrdinalIgnoreCase)), - StaleSources = records.Count(r => string.Equals(r.FreshnessStatus, "stale", StringComparison.OrdinalIgnoreCase)), - UnavailableSources = records.Count(r => string.Equals(r.FreshnessStatus, "unavailable", StringComparison.OrdinalIgnoreCase)), - DisabledSources = records.Count(r => !r.Enabled), - ConflictingSources = 0, + TotalSources = items.Count, + HealthySources = items.Count(static r => string.Equals(r.FreshnessStatus, "healthy", StringComparison.OrdinalIgnoreCase)), + WarningSources = items.Count(static r => string.Equals(r.FreshnessStatus, "warning", StringComparison.OrdinalIgnoreCase)), + StaleSources = items.Count(static r => string.Equals(r.FreshnessStatus, "stale", StringComparison.OrdinalIgnoreCase)), + UnavailableSources = items.Count(static r => string.Equals(r.FreshnessStatus, "unavailable", StringComparison.OrdinalIgnoreCase)), + DisabledSources = items.Count(static r => !r.Enabled), + ConflictingSources = records + .GroupBy(static record => SourceKeyAliases.Normalize(record.SourceKey), StringComparer.OrdinalIgnoreCase) + .Count(static group => group.Count() > 1), DataAsOf = timeProvider.GetUtcNow() }; @@ -93,7 +97,6 @@ internal static class AdvisorySourceEndpointExtensions HttpContext httpContext, string id, [FromServices] IAdvisorySourceReadRepository readRepository, - [FromServices] ISourceRepository sourceRepository, [FromServices] IMemoryCache cache, TimeProvider timeProvider, CancellationToken cancellationToken) => @@ -104,40 +107,43 @@ internal static class AdvisorySourceEndpointExtensions } id = id.Trim(); - var cacheKey = $"advisory-sources:freshness:{id}"; + var cacheKey = Guid.TryParse(id, out _) ? $"advisory-sources:freshness:{id}" : $"advisory-sources:freshness:{SourceKeyAliases.Normalize(id)}"; if (cache.TryGetValue(cacheKey, out AdvisorySourceFreshnessResponse? cached)) { return HttpResults.Ok(cached); } - AdvisorySourceFreshnessRecord? record = null; + AdvisorySourceListItem? item = null; if (Guid.TryParse(id, out var sourceId)) { - record = await readRepository.GetBySourceIdAsync(sourceId, cancellationToken).ConfigureAwait(false); + var record = await readRepository.GetBySourceIdAsync(sourceId, cancellationToken).ConfigureAwait(false); + if (record is not null) + { + item = BuildListItems([record], includeDisabled: true).Single(); + } } else { - var source = await sourceRepository.GetByKeyAsync(id, cancellationToken).ConfigureAwait(false); - if (source is not null) - { - record = await readRepository.GetBySourceIdAsync(source.Id, cancellationToken).ConfigureAwait(false); - } + var normalizedId = SourceKeyAliases.Normalize(id); + var records = await readRepository.ListAsync(includeDisabled: true, cancellationToken).ConfigureAwait(false); + item = BuildListItems(records, includeDisabled: true) + .FirstOrDefault(source => string.Equals(source.SourceKey, normalizedId, StringComparison.OrdinalIgnoreCase)); } - if (record is null) + if (item is null) { return HttpResults.NotFound(new { error = "advisory_source_not_found", id }); } var response = new AdvisorySourceFreshnessResponse { - Source = MapListItem(record), - LastSyncAt = record.LastSyncAt, - LastSuccessAt = record.LastSuccessAt, - LastError = record.LastError, - SyncCount = record.SyncCount, - ErrorCount = record.ErrorCount, + Source = item, + LastSyncAt = item.LastSyncAt, + LastSuccessAt = item.LastSuccessAt, + LastError = item.LastError, + SyncCount = item.SyncCount, + ErrorCount = item.ErrorCount, DataAsOf = timeProvider.GetUtcNow() }; @@ -152,17 +158,56 @@ internal static class AdvisorySourceEndpointExtensions .RequireAuthorization(AdvisoryReadPolicy); } - private static AdvisorySourceListItem MapListItem(AdvisorySourceFreshnessRecord record) + private static List BuildListItems( + IReadOnlyList records, + bool includeDisabled) + { + return records + .GroupBy(static record => SourceKeyAliases.Normalize(record.SourceKey), StringComparer.OrdinalIgnoreCase) + .Select(MapGroupedRecord) + .Where(item => includeDisabled || item.Enabled) + .OrderBy(static item => item.SourceName, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + private static AdvisorySourceListItem MapGroupedRecord(IGrouping group) + { + var normalizedKey = group.Key; + var records = group.ToList(); + var representative = records + .OrderByDescending(static record => record.LastSuccessAt ?? record.LastSyncAt ?? DateTimeOffset.MinValue) + .ThenByDescending(static record => record.Enabled) + .ThenByDescending(record => string.Equals(record.SourceKey, normalizedKey, StringComparison.OrdinalIgnoreCase)) + .ThenBy(static record => record.Priority) + .First(); + + var definition = SourceDefinitions.FindById(normalizedKey); + return MapListItem( + representative, + normalizedKey, + definition?.DisplayName ?? representative.SourceName, + definition?.Id ?? representative.SourceFamily, + definition?.DefaultPriority ?? representative.Priority, + records.Any(static record => record.Enabled)); + } + + private static AdvisorySourceListItem MapListItem( + AdvisorySourceFreshnessRecord record, + string sourceKey, + string sourceName, + string sourceFamily, + int priority, + bool enabled) { return new AdvisorySourceListItem { SourceId = record.SourceId, - SourceKey = record.SourceKey, - SourceName = record.SourceName, - SourceFamily = record.SourceFamily, + SourceKey = sourceKey, + SourceName = sourceName, + SourceFamily = sourceFamily, SourceUrl = record.SourceUrl, - Priority = record.Priority, - Enabled = record.Enabled, + Priority = priority, + Enabled = enabled, LastSyncAt = record.LastSyncAt, LastSuccessAt = record.LastSuccessAt, FreshnessAgeSeconds = record.FreshnessAgeSeconds, @@ -172,6 +217,10 @@ internal static class AdvisorySourceEndpointExtensions LastError = record.LastError, SyncCount = record.SyncCount, ErrorCount = record.ErrorCount, + SourceDocumentCount = record.SourceDocumentCount, + CanonicalAdvisoryCount = record.CanonicalAdvisoryCount, + CveCount = record.CveCount, + VexDocumentCount = record.VexDocumentCount, TotalAdvisories = record.TotalAdvisories, SignedAdvisories = record.SignedAdvisories, UnsignedAdvisories = record.UnsignedAdvisories, @@ -205,6 +254,10 @@ public sealed record AdvisorySourceListItem public string? LastError { get; init; } public long SyncCount { get; init; } public int ErrorCount { get; init; } + public long SourceDocumentCount { get; init; } + public long CanonicalAdvisoryCount { get; init; } + public long CveCount { get; init; } + public long VexDocumentCount { get; init; } public long TotalAdvisories { get; init; } public long SignedAdvisories { get; init; } public long UnsignedAdvisories { get; init; } diff --git a/src/Concelier/StellaOps.Concelier.WebService/Extensions/AirGapEndpointExtensions.cs b/src/Concelier/StellaOps.Concelier.WebService/Extensions/AirGapEndpointExtensions.cs index e5a938801..52bc45a8f 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Extensions/AirGapEndpointExtensions.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Extensions/AirGapEndpointExtensions.cs @@ -3,6 +3,7 @@ using HttpResults = Microsoft.AspNetCore.Http.Results; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; +using StellaOps.Audit.Emission; using StellaOps.Auth.ServerIntegration.Tenancy; using StellaOps.Concelier.Core.AirGap; using StellaOps.Concelier.Core.AirGap.Models; @@ -97,7 +98,8 @@ internal static class AirGapEndpointExtensions .WithName("RegisterAirGapSource") .WithSummary("Register a new air-gap bundle source") .WithDescription("Registers a new bundle source in the air-gap source registry. The source ID must be unique and is used to identify the source in subsequent operations.") - .RequireAuthorization("Concelier.Advisories.Ingest"); + .RequireAuthorization("Concelier.Advisories.Ingest") + .Audited("concelier", "create", resourceType: "airgap_source"); // GET /api/v1/concelier/airgap/sources/{sourceId} - Get specific source group.MapGet("/sources/{sourceId}", ( @@ -149,7 +151,8 @@ internal static class AirGapEndpointExtensions .WithName("UnregisterAirGapSource") .WithSummary("Unregister an air-gap bundle source") .WithDescription("Removes a bundle source from the air-gap registry. This does not delete any previously downloaded bundles.") - .RequireAuthorization("Concelier.Advisories.Ingest"); + .RequireAuthorization("Concelier.Advisories.Ingest") + .Audited("concelier", "delete", resourceType: "airgap_source"); // POST /api/v1/concelier/airgap/sources/{sourceId}/validate - Validate source group.MapPost("/sources/{sourceId}/validate", async ( @@ -173,7 +176,8 @@ internal static class AirGapEndpointExtensions .WithName("ValidateAirGapSource") .WithSummary("Validate an air-gap bundle source") .WithDescription("Runs connectivity and integrity checks against a registered bundle source to verify it is reachable and correctly configured.") - .RequireAuthorization("Concelier.Advisories.Ingest"); + .RequireAuthorization("Concelier.Advisories.Ingest") + .Audited("concelier", "verify", resourceType: "airgap_source"); // GET /api/v1/concelier/airgap/status - Sealed-mode status group.MapGet("/status", ( @@ -285,7 +289,8 @@ internal static class AirGapEndpointExtensions .WithName("ImportAirGapBundle") .WithSummary("Import an air-gap bundle with timeline event") .WithDescription("Imports a specific bundle from the catalog into the tenant's advisory database and emits a timeline event recording the import actor, scope, and statistics.") - .RequireAuthorization("Concelier.Advisories.Ingest"); + .RequireAuthorization("Concelier.Advisories.Ingest") + .Audited("concelier", "import", resourceType: "airgap_bundle"); } } diff --git a/src/Concelier/StellaOps.Concelier.WebService/Extensions/CanonicalAdvisoryEndpointExtensions.cs b/src/Concelier/StellaOps.Concelier.WebService/Extensions/CanonicalAdvisoryEndpointExtensions.cs index 9e9d46737..5fc0718e0 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Extensions/CanonicalAdvisoryEndpointExtensions.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Extensions/CanonicalAdvisoryEndpointExtensions.cs @@ -8,6 +8,7 @@ using HttpResults = Microsoft.AspNetCore.Http.Results; using Microsoft.AspNetCore.Mvc; +using StellaOps.Audit.Emission; using StellaOps.Auth.ServerIntegration.Tenancy; using StellaOps.Concelier.Core.Canonical; using StellaOps.Concelier.Interest; @@ -192,7 +193,8 @@ internal static class CanonicalAdvisoryEndpointExtensions .Produces(StatusCodes.Status200OK) .Produces(StatusCodes.Status409Conflict) .Produces(StatusCodes.Status400BadRequest) - .RequireAuthorization(CanonicalIngestPolicy); + .RequireAuthorization(CanonicalIngestPolicy) + .Audited("concelier", "create", resourceType: "canonical_advisory"); // POST /api/v1/canonical/ingest/{source}/batch - Batch ingest advisories group.MapPost("/ingest/{source}/batch", async ( @@ -255,7 +257,8 @@ internal static class CanonicalAdvisoryEndpointExtensions .WithDescription("Ingests a batch of raw advisories from the named source into the canonical merge pipeline. Returns per-item merge decisions and a summary with total created, merged, duplicate, and conflict counts.") .Produces(StatusCodes.Status200OK) .Produces(StatusCodes.Status400BadRequest) - .RequireAuthorization(CanonicalIngestPolicy); + .RequireAuthorization(CanonicalIngestPolicy) + .Audited("concelier", "import", resourceType: "canonical_advisory_batch"); // PATCH /api/v1/canonical/{id}/status - Update canonical status group.MapPatch("/{id:guid}/status", async ( @@ -279,7 +282,8 @@ internal static class CanonicalAdvisoryEndpointExtensions .WithDescription("Updates the lifecycle status (Active, Disputed, Suppressed, Withdrawn) of a canonical advisory. Used by triage workflows to manage advisory state.") .Produces(StatusCodes.Status200OK) .Produces(StatusCodes.Status400BadRequest) - .RequireAuthorization(CanonicalIngestPolicy); + .RequireAuthorization(CanonicalIngestPolicy) + .Audited("concelier", "update", resourceType: "canonical_advisory"); // GET /api/v1/canonical/{id}/provenance - Get provenance scopes for canonical group.MapGet("/{id:guid}/provenance", async ( diff --git a/src/Concelier/StellaOps.Concelier.WebService/Extensions/FederationEndpointExtensions.cs b/src/Concelier/StellaOps.Concelier.WebService/Extensions/FederationEndpointExtensions.cs index 4c4a22373..334d44a43 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Extensions/FederationEndpointExtensions.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Extensions/FederationEndpointExtensions.cs @@ -2,6 +2,7 @@ using HttpResults = Microsoft.AspNetCore.Http.Results; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; +using StellaOps.Audit.Emission; using StellaOps.Auth.ServerIntegration.Tenancy; using StellaOps.Concelier.Federation.Export; using StellaOps.Concelier.Federation.Import; @@ -244,7 +245,8 @@ internal static class FederationEndpointExtensions .ProducesProblem(422) .ProducesProblem(503) .DisableAntiforgery() - .RequireAuthorization("Concelier.Advisories.Ingest"); + .RequireAuthorization("Concelier.Advisories.Ingest") + .Audited("concelier", "import", resourceType: "federation_bundle"); // POST /api/v1/federation/import/validate - Validate bundle without importing group.MapPost("/import/validate", async ( @@ -280,7 +282,8 @@ internal static class FederationEndpointExtensions .Produces(200) .ProducesProblem(503) .DisableAntiforgery() - .RequireAuthorization("Concelier.Advisories.Ingest"); + .RequireAuthorization("Concelier.Advisories.Ingest") + .Audited("concelier", "verify", resourceType: "federation_bundle"); // POST /api/v1/federation/import/preview - Preview import group.MapPost("/import/preview", async ( @@ -473,7 +476,8 @@ internal static class FederationEndpointExtensions .Produces(200) .ProducesProblem(400) .ProducesProblem(503) - .RequireAuthorization("Concelier.Advisories.Ingest"); + .RequireAuthorization("Concelier.Advisories.Ingest") + .Audited("concelier", "update", resourceType: "federation_site_policy"); } } diff --git a/src/Concelier/StellaOps.Concelier.WebService/Extensions/FeedMirrorManagementEndpoints.cs b/src/Concelier/StellaOps.Concelier.WebService/Extensions/FeedMirrorManagementEndpoints.cs index d5caf054d..a36352b3e 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Extensions/FeedMirrorManagementEndpoints.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Extensions/FeedMirrorManagementEndpoints.cs @@ -69,8 +69,8 @@ internal static class FeedMirrorManagementEndpoints .RequireAuthorization("Concelier.Advisories.Ingest") .RequireTenant(); - imports.MapPost("/validate", ValidateImport).WithName("ValidateAirGapImport").RequireAuthorization("Concelier.Advisories.Ingest"); - imports.MapPost("/", StartImport).WithName("StartAirGapImport").RequireAuthorization("Concelier.Advisories.Ingest"); + imports.MapPost("/validate", ValidateImport).WithName("ValidateAirGapImport").RequireAuthorization("Concelier.Advisories.Ingest").Audited("concelier", "verify", resourceType: "airgap_import"); + imports.MapPost("/", StartImport).WithName("StartAirGapImport").RequireAuthorization("Concelier.Advisories.Ingest").Audited("concelier", "import", resourceType: "airgap_import"); imports.MapGet("/{importId}", GetImportProgress).WithName("GetAirGapImportProgress").RequireAuthorization("Concelier.Advisories.Read"); var versionLocks = app.MapGroup("/api/v1/concelier/version-locks") @@ -80,8 +80,8 @@ internal static class FeedMirrorManagementEndpoints versionLocks.MapGet(string.Empty, ListVersionLocks).WithName("ListVersionLocks").RequireAuthorization("Concelier.Advisories.Read"); versionLocks.MapGet("/{feedType}", GetVersionLock).WithName("GetVersionLock").RequireAuthorization("Concelier.Advisories.Read"); - versionLocks.MapPut("/{feedType}", SetVersionLock).WithName("SetVersionLock").RequireAuthorization("Concelier.Advisories.Ingest"); - versionLocks.MapDelete("/{lockId}", RemoveVersionLock).WithName("RemoveVersionLock").RequireAuthorization("Concelier.Advisories.Ingest"); + versionLocks.MapPut("/{feedType}", SetVersionLock).WithName("SetVersionLock").RequireAuthorization("Concelier.Advisories.Ingest").Audited("concelier", "update", resourceType: "version_lock"); + versionLocks.MapDelete("/{lockId}", RemoveVersionLock).WithName("RemoveVersionLock").RequireAuthorization("Concelier.Advisories.Ingest").Audited("concelier", "delete", resourceType: "version_lock"); app.MapGet("/api/v1/concelier/offline-status", GetOfflineSyncStatus) .WithTags("OfflineStatus") diff --git a/src/Concelier/StellaOps.Concelier.WebService/Extensions/FeedSnapshotEndpointExtensions.cs b/src/Concelier/StellaOps.Concelier.WebService/Extensions/FeedSnapshotEndpointExtensions.cs index 0cfa029b3..0551e02e6 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Extensions/FeedSnapshotEndpointExtensions.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Extensions/FeedSnapshotEndpointExtensions.cs @@ -12,6 +12,7 @@ using HttpResults = Microsoft.AspNetCore.Http.Results; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; +using StellaOps.Audit.Emission; using StellaOps.Auth.ServerIntegration.Tenancy; using StellaOps.Concelier.WebService.Options; using StellaOps.Concelier.WebService.Results; @@ -42,7 +43,8 @@ internal static class FeedSnapshotEndpointExtensions .WithName("CreateFeedSnapshot") .WithSummary("Create an atomic feed snapshot") .WithDescription("Creates an atomic, point-in-time snapshot of all registered feed sources, computing per-source digests and a composite digest for deterministic replay and offline-first bundle generation. Optionally scoped to a subset of sources via the sources field.") - .RequireAuthorization("Concelier.Advisories.Ingest"); + .RequireAuthorization("Concelier.Advisories.Ingest") + .Audited("concelier", "create", resourceType: "feed_snapshot"); // GET /api/v1/feeds/snapshot - List available snapshots group.MapGet("/", ListSnapshotsAsync) @@ -70,7 +72,8 @@ internal static class FeedSnapshotEndpointExtensions .WithName("ImportFeedSnapshot") .WithSummary("Import feed snapshot bundle") .WithDescription("Imports a snapshot bundle from a compressed archive uploaded as a multipart file, optionally validating per-source digests before registering the snapshot. The resulting snapshot ID is returned in the Location header.") - .RequireAuthorization("Concelier.Advisories.Ingest"); + .RequireAuthorization("Concelier.Advisories.Ingest") + .Audited("concelier", "import", resourceType: "feed_snapshot"); // GET /api/v1/feeds/snapshot/{snapshotId}/validate - Validate snapshot group.MapGet("/{snapshotId}/validate", ValidateSnapshotAsync) diff --git a/src/Concelier/StellaOps.Concelier.WebService/Extensions/InterestScoreEndpointExtensions.cs b/src/Concelier/StellaOps.Concelier.WebService/Extensions/InterestScoreEndpointExtensions.cs index 3004c58fe..2ef21653d 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Extensions/InterestScoreEndpointExtensions.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Extensions/InterestScoreEndpointExtensions.cs @@ -8,6 +8,7 @@ using HttpResults = Microsoft.AspNetCore.Http.Results; using Microsoft.AspNetCore.Mvc; +using StellaOps.Audit.Emission; using StellaOps.Auth.ServerIntegration.Tenancy; using StellaOps.Concelier.Interest; using StellaOps.Concelier.Interest.Models; @@ -125,7 +126,8 @@ internal static class InterestScoreEndpointExtensions .WithSummary("Compute and update interest score for a canonical advisory") .WithDescription("Triggers an on-demand interest score computation for a single canonical advisory and persists the result. Useful for forcing a score refresh after SBOM registration, reachability updates, or manual investigation.") .Produces(StatusCodes.Status200OK) - .RequireAuthorization(ScoreAdminPolicy); + .RequireAuthorization(ScoreAdminPolicy) + .Audited("concelier", "execute", resourceType: "interest_score"); // POST /api/v1/scores/recalculate - Admin endpoint to trigger full recalculation group.MapPost("/scores/recalculate", async ( @@ -157,7 +159,8 @@ internal static class InterestScoreEndpointExtensions .WithSummary("Trigger interest score recalculation (full or batch)") .WithDescription("Enqueues an interest score recalculation for either a specific set of canonical IDs (batch mode) or all advisories (full mode). Returns 202 Accepted immediately; actual updates occur asynchronously in the scoring background job.") .Produces(StatusCodes.Status202Accepted) - .RequireAuthorization(ScoreAdminPolicy); + .RequireAuthorization(ScoreAdminPolicy) + .Audited("concelier", "execute", resourceType: "interest_score"); // POST /api/v1/scores/degrade - Admin endpoint to run stub degradation group.MapPost("/scores/degrade", async ( @@ -182,7 +185,8 @@ internal static class InterestScoreEndpointExtensions .WithSummary("Degrade low-interest advisories to stubs") .WithDescription("Downgrades all canonical advisories whose interest score falls below the specified threshold (or the configured default) to stub representation, reducing storage footprint. Returns the count of advisories degraded.") .Produces(StatusCodes.Status200OK) - .RequireAuthorization(ScoreAdminPolicy); + .RequireAuthorization(ScoreAdminPolicy) + .Audited("concelier", "execute", resourceType: "interest_score_stub"); // POST /api/v1/scores/restore - Admin endpoint to restore stubs group.MapPost("/scores/restore", async ( @@ -207,7 +211,8 @@ internal static class InterestScoreEndpointExtensions .WithSummary("Restore stubs with increased interest scores") .WithDescription("Promotes stub advisories whose interest score now exceeds the specified restoration threshold back to full canonical representation. Typically triggered after new SBOM registrations or reachability discoveries raise scores above the stub cutoff.") .Produces(StatusCodes.Status200OK) - .RequireAuthorization(ScoreAdminPolicy); + .RequireAuthorization(ScoreAdminPolicy) + .Audited("concelier", "execute", resourceType: "interest_score_stub"); } private static InterestScoreResponse MapToResponse(InterestScore score) => new() diff --git a/src/Concelier/StellaOps.Concelier.WebService/Extensions/InternalSetupSourceEndpointExtensions.cs b/src/Concelier/StellaOps.Concelier.WebService/Extensions/InternalSetupSourceEndpointExtensions.cs index 01fcf9088..95f3c130a 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Extensions/InternalSetupSourceEndpointExtensions.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Extensions/InternalSetupSourceEndpointExtensions.cs @@ -1,4 +1,5 @@ using Microsoft.AspNetCore.Mvc; +using StellaOps.Audit.Emission; using StellaOps.Concelier.WebService.Services; namespace StellaOps.Concelier.WebService.Extensions; @@ -39,7 +40,8 @@ internal static class InternalSetupSourceEndpointExtensions Status = StatusCodes.Status400BadRequest }); } - }); + }) + .Audited("concelier", "verify", resourceType: "advisory_source_bootstrap"); group.MapPost("/apply", async Task ( HttpContext context, @@ -68,7 +70,8 @@ internal static class InternalSetupSourceEndpointExtensions Status = StatusCodes.Status400BadRequest }); } - }); + }) + .Audited("concelier", "update", resourceType: "advisory_source_bootstrap"); } private static IResult? ValidateBootstrapRequest(HttpContext context, IConfiguration configuration) diff --git a/src/Concelier/StellaOps.Concelier.WebService/Extensions/JobRegistrationExtensions.cs b/src/Concelier/StellaOps.Concelier.WebService/Extensions/JobRegistrationExtensions.cs index 2d4ff4587..a9c4ade53 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Extensions/JobRegistrationExtensions.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Extensions/JobRegistrationExtensions.cs @@ -23,6 +23,14 @@ internal static class JobRegistrationExtensions private static readonly IReadOnlyList BaseBuiltInJobs = new List { + new("source:cve:fetch", "StellaOps.Concelier.Connector.Cve.CveFetchJob", "StellaOps.Concelier.Connector.Cve", TimeSpan.FromMinutes(12), TimeSpan.FromMinutes(6), "3 */4 * * *"), + new("source:cve:parse", "StellaOps.Concelier.Connector.Cve.CveParseJob", "StellaOps.Concelier.Connector.Cve", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(6), "13 */4 * * *"), + new("source:cve:map", "StellaOps.Concelier.Connector.Cve.CveMapJob", "StellaOps.Concelier.Connector.Cve", TimeSpan.FromMinutes(20), TimeSpan.FromMinutes(6), "23 */4 * * *"), + + new("source:nvd:fetch", "StellaOps.Concelier.Connector.Nvd.NvdFetchJob", "StellaOps.Concelier.Connector.Nvd", TimeSpan.FromMinutes(12), TimeSpan.FromMinutes(6), "6 */4 * * *"), + new("source:nvd:parse", "StellaOps.Concelier.Connector.Nvd.NvdParseJob", "StellaOps.Concelier.Connector.Nvd", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(6), "16 */4 * * *"), + new("source:nvd:map", "StellaOps.Concelier.Connector.Nvd.NvdMapJob", "StellaOps.Concelier.Connector.Nvd", TimeSpan.FromMinutes(20), TimeSpan.FromMinutes(6), "26 */4 * * *"), + new("source:redhat:fetch", "StellaOps.Concelier.Connector.Distro.RedHat.RedHatFetchJob", "StellaOps.Concelier.Connector.Distro.RedHat", TimeSpan.FromMinutes(12), TimeSpan.FromMinutes(6), "0 */4 * * *"), new("source:redhat:parse", "StellaOps.Concelier.Connector.Distro.RedHat.RedHatParseJob", "StellaOps.Concelier.Connector.Distro.RedHat", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(6), "10 */4 * * *"), new("source:redhat:map", "StellaOps.Concelier.Connector.Distro.RedHat.RedHatMapJob", "StellaOps.Concelier.Connector.Distro.RedHat", TimeSpan.FromMinutes(20), TimeSpan.FromMinutes(6), "20 */4 * * *"), @@ -91,21 +99,27 @@ internal static class JobRegistrationExtensions new("source:suse:map", "StellaOps.Concelier.Connector.Distro.Suse.SuseMapJob", "StellaOps.Concelier.Connector.Distro.Suse", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), // ── CERT-BUND (fetch only) ── - new("source:cert-bund:fetch", "StellaOps.Concelier.Connector.CertBund.CertBundFetchJob", "StellaOps.Concelier.Connector.CertBund", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:cert-de:fetch", "StellaOps.Concelier.Connector.CertBund.CertBundFetchJob", "StellaOps.Concelier.Connector.CertBund", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - // ── CERT-CC (fetch only) ── + // ── CERT-CC ── new("source:cert-cc:fetch", "StellaOps.Concelier.Connector.CertCc.CertCcFetchJob", "StellaOps.Concelier.Connector.CertCc", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:cert-cc:parse", "StellaOps.Concelier.Connector.CertCc.CertCcParseJob", "StellaOps.Concelier.Connector.CertCc", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:cert-cc:map", "StellaOps.Concelier.Connector.CertCc.CertCcMapJob", "StellaOps.Concelier.Connector.CertCc", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), // ── ACSC (Australian Cyber Security Centre) ── new("source:auscert:fetch", "StellaOps.Concelier.Connector.Acsc.AcscFetchJob", "StellaOps.Concelier.Connector.Acsc", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), new("source:auscert:parse", "StellaOps.Concelier.Connector.Acsc.AcscParseJob", "StellaOps.Concelier.Connector.Acsc", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), new("source:auscert:map", "StellaOps.Concelier.Connector.Acsc.AcscMapJob", "StellaOps.Concelier.Connector.Acsc", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - // ── CCCS (Canadian Centre for Cyber Security, fetch only) ── + // ── CCCS (Canadian Centre for Cyber Security) ── new("source:cccs:fetch", "StellaOps.Concelier.Connector.Cccs.CccsFetchJob", "StellaOps.Concelier.Connector.Cccs", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:cccs:parse", "StellaOps.Concelier.Connector.Cccs.CccsParseJob", "StellaOps.Concelier.Connector.Cccs", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:cccs:map", "StellaOps.Concelier.Connector.Cccs.CccsMapJob", "StellaOps.Concelier.Connector.Cccs", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - // ── KISA (Korea, fetch only) ── - new("source:kisa:fetch", "StellaOps.Concelier.Connector.Kisa.KisaFetchJob", "StellaOps.Concelier.Connector.Kisa", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + // ── KISA (Korea) ── + new("source:krcert:fetch", "StellaOps.Concelier.Connector.Kisa.KisaFetchJob", "StellaOps.Concelier.Connector.Kisa", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:krcert:parse", "StellaOps.Concelier.Connector.Kisa.KisaParseJob", "StellaOps.Concelier.Connector.Kisa", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:krcert:map", "StellaOps.Concelier.Connector.Kisa.KisaMapJob", "StellaOps.Concelier.Connector.Kisa", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), // ── RU-BDU (Russian FSTEC) ── new("source:fstec-bdu:fetch", "StellaOps.Concelier.Connector.Ru.Bdu.RuBduFetchJob", "StellaOps.Concelier.Connector.Ru.Bdu", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), @@ -122,13 +136,25 @@ internal static class JobRegistrationExtensions new("source:apple:parse", "StellaOps.Concelier.Connector.Vndr.Apple.AppleParseJob", "StellaOps.Concelier.Connector.Vndr.Apple", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), new("source:apple:map", "StellaOps.Concelier.Connector.Vndr.Apple.AppleMapJob", "StellaOps.Concelier.Connector.Vndr.Apple", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + // ── Vendor: Adobe ── + new("source:adobe:fetch", "StellaOps.Concelier.Connector.Vndr.Adobe.AdobeFetchJob", "StellaOps.Concelier.Connector.Vndr.Adobe", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:adobe:parse", "StellaOps.Concelier.Connector.Vndr.Adobe.AdobeParseJob", "StellaOps.Concelier.Connector.Vndr.Adobe", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:adobe:map", "StellaOps.Concelier.Connector.Vndr.Adobe.AdobeMapJob", "StellaOps.Concelier.Connector.Vndr.Adobe", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + // ── Vendor: Chromium ── + new("source:chromium:fetch", "StellaOps.Concelier.Connector.Vndr.Chromium.ChromiumFetchJob", "StellaOps.Concelier.Connector.Vndr.Chromium", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:chromium:parse", "StellaOps.Concelier.Connector.Vndr.Chromium.ChromiumParseJob", "StellaOps.Concelier.Connector.Vndr.Chromium", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:chromium:map", "StellaOps.Concelier.Connector.Vndr.Chromium.ChromiumMapJob", "StellaOps.Concelier.Connector.Vndr.Chromium", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + // ── Vendor: Cisco ── new("source:cisco:fetch", "StellaOps.Concelier.Connector.Vndr.Cisco.CiscoFetchJob", "StellaOps.Concelier.Connector.Vndr.Cisco", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), new("source:cisco:parse", "StellaOps.Concelier.Connector.Vndr.Cisco.CiscoParseJob", "StellaOps.Concelier.Connector.Vndr.Cisco", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), new("source:cisco:map", "StellaOps.Concelier.Connector.Vndr.Cisco.CiscoMapJob", "StellaOps.Concelier.Connector.Vndr.Cisco", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - // ── Vendor: Microsoft MSRC (fetch only) ── + // ── Vendor: Microsoft MSRC ── new("source:microsoft:fetch", "StellaOps.Concelier.Connector.Vndr.Msrc.MsrcFetchJob", "StellaOps.Concelier.Connector.Vndr.Msrc", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:microsoft:parse", "StellaOps.Concelier.Connector.Vndr.Msrc.MsrcParseJob", "StellaOps.Concelier.Connector.Vndr.Msrc", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:microsoft:map", "StellaOps.Concelier.Connector.Vndr.Msrc.MsrcMapJob", "StellaOps.Concelier.Connector.Vndr.Msrc", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), // ── ICS-CISA ── new("source:us-cert:fetch", "StellaOps.Concelier.Connector.Ics.Cisa.IcsCisaFetchJob", "StellaOps.Concelier.Connector.Ics.Cisa", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), diff --git a/src/Concelier/StellaOps.Concelier.WebService/Extensions/MirrorDomainManagementEndpointExtensions.cs b/src/Concelier/StellaOps.Concelier.WebService/Extensions/MirrorDomainManagementEndpointExtensions.cs index 41999b957..dffa80a8a 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Extensions/MirrorDomainManagementEndpointExtensions.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Extensions/MirrorDomainManagementEndpointExtensions.cs @@ -5,7 +5,9 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; using Npgsql; using NpgsqlTypes; +using StellaOps.Audit.Emission; using StellaOps.Auth.ServerIntegration.Tenancy; +using StellaOps.Concelier.Core.Sources; using StellaOps.Concelier.Persistence.Postgres; using StellaOps.Concelier.Persistence.Postgres.Models; using StellaOps.Concelier.Persistence.Postgres.Repositories; @@ -54,7 +56,8 @@ internal static class MirrorDomainManagementEndpointExtensions .WithSummary("Update mirror mode, signing, and refresh settings") .WithDescription("Updates persisted mirror mode, signing, and refresh settings.") .Produces(StatusCodes.Status200OK) - .RequireAuthorization(MirrorManagePolicy); + .RequireAuthorization(MirrorManagePolicy) + .Audited("concelier", "update", resourceType: "mirror_config"); // GET /domains — list all configured mirror domains group.MapGet("/health", ([FromServices] IMirrorDomainStore domainStore) => @@ -86,7 +89,7 @@ internal static class MirrorDomainManagementEndpointExtensions .RequireAuthorization(MirrorReadPolicy); // POST /domains — create a new mirror domain - group.MapPost("/domains", async ([FromBody] CreateMirrorDomainRequest request, [FromServices] IMirrorDomainStore domainStore, [FromServices] ISourceRepository sourceRepository, CancellationToken ct) => + group.MapPost("/domains", async ([FromBody] CreateMirrorDomainRequest request, [FromServices] IMirrorDomainStore domainStore, [FromServices] ISourceRegistry sourceRegistry, CancellationToken ct) => { var domainId = NormalizeDomainId(request.DomainId ?? request.Id); if (string.IsNullOrWhiteSpace(domainId)) @@ -105,8 +108,7 @@ internal static class MirrorDomainManagementEndpointExtensions return HttpResults.BadRequest(new { error = "source_ids_required" }); } - var knownSources = await sourceRepository.ListAsync(enabled: null, cancellationToken: ct).ConfigureAwait(false); - var invalidSourceIds = FindUnknownSourceIds(sourceIds, knownSources); + var invalidSourceIds = FindUnknownSourceIds(sourceIds, sourceRegistry); if (invalidSourceIds.Count > 0) { return HttpResults.BadRequest(new { error = "unknown_source_ids", sourceIds = invalidSourceIds }); @@ -138,7 +140,8 @@ internal static class MirrorDomainManagementEndpointExtensions .WithSummary("Create a new mirror domain") .WithDescription("Creates a persisted mirror domain backed by real Concelier source configuration.") .Produces(StatusCodes.Status201Created) - .RequireAuthorization(MirrorManagePolicy); + .RequireAuthorization(MirrorManagePolicy) + .Audited("concelier", "create", resourceType: "mirror_domain"); // GET /domains/{domainId} — get domain detail group.MapGet("/domains/{domainId}", (string domainId, [FromServices] IMirrorDomainStore domainStore) => @@ -156,7 +159,7 @@ internal static class MirrorDomainManagementEndpointExtensions .RequireAuthorization(MirrorReadPolicy); // PUT /domains/{domainId} — update domain - group.MapPut("/domains/{domainId}", async (string domainId, [FromBody] UpdateMirrorDomainRequest request, [FromServices] IMirrorDomainStore domainStore, [FromServices] ISourceRepository sourceRepository, CancellationToken ct) => + group.MapPut("/domains/{domainId}", async (string domainId, [FromBody] UpdateMirrorDomainRequest request, [FromServices] IMirrorDomainStore domainStore, [FromServices] ISourceRegistry sourceRegistry, CancellationToken ct) => { var existing = domainStore.GetDomain(domainId); if (existing is null) @@ -170,8 +173,7 @@ internal static class MirrorDomainManagementEndpointExtensions return HttpResults.BadRequest(new { error = "source_ids_required" }); } - var knownSources = await sourceRepository.ListAsync(enabled: null, cancellationToken: ct).ConfigureAwait(false); - var invalidSourceIds = FindUnknownSourceIds(resolvedSourceIds, knownSources); + var invalidSourceIds = FindUnknownSourceIds(resolvedSourceIds, sourceRegistry); if (invalidSourceIds.Count > 0) { return HttpResults.BadRequest(new { error = "unknown_source_ids", sourceIds = invalidSourceIds }); @@ -209,7 +211,8 @@ internal static class MirrorDomainManagementEndpointExtensions .WithDescription("Updates an existing persisted mirror domain.") .Produces(StatusCodes.Status200OK) .Produces(StatusCodes.Status404NotFound) - .RequireAuthorization(MirrorManagePolicy); + .RequireAuthorization(MirrorManagePolicy) + .Audited("concelier", "update", resourceType: "mirror_domain"); // DELETE /domains/{domainId} — remove domain group.MapDelete("/domains/{domainId}", async (string domainId, [FromServices] IMirrorDomainStore domainStore, [FromServices] IOptionsMonitor optionsMonitor, CancellationToken ct) => @@ -230,7 +233,8 @@ internal static class MirrorDomainManagementEndpointExtensions .WithDescription("Deletes a persisted mirror domain and removes its generated artifacts.") .Produces(StatusCodes.Status204NoContent) .Produces(StatusCodes.Status404NotFound) - .RequireAuthorization(MirrorManagePolicy); + .RequireAuthorization(MirrorManagePolicy) + .Audited("concelier", "delete", resourceType: "mirror_domain"); // POST /domains/{domainId}/exports — add export to domain group.MapGet("/domains/{domainId}/config", (string domainId, [FromServices] IMirrorDomainStore domainStore) => @@ -302,7 +306,8 @@ internal static class MirrorDomainManagementEndpointExtensions .WithDescription("Adds or replaces an export definition on a persisted mirror domain.") .Produces(StatusCodes.Status200OK) .Produces(StatusCodes.Status404NotFound) - .RequireAuthorization(MirrorManagePolicy); + .RequireAuthorization(MirrorManagePolicy) + .Audited("concelier", "create", resourceType: "mirror_export"); // DELETE /domains/{domainId}/exports/{exportKey} — remove export group.MapDelete("/domains/{domainId}/exports/{exportKey}", async (string domainId, string exportKey, [FromServices] IMirrorDomainStore domainStore, CancellationToken ct) => @@ -328,7 +333,8 @@ internal static class MirrorDomainManagementEndpointExtensions .WithDescription("Removes an export definition from a persisted mirror domain.") .Produces(StatusCodes.Status200OK) .Produces(StatusCodes.Status404NotFound) - .RequireAuthorization(MirrorManagePolicy); + .RequireAuthorization(MirrorManagePolicy) + .Audited("concelier", "delete", resourceType: "mirror_export"); // POST /domains/{domainId}/generate — trigger bundle generation group.MapPost("/domains/{domainId}/generate", async (string domainId, [FromServices] IMirrorDomainStore domainStore, [FromServices] ConcelierDataSource dataSource, [FromServices] IOptionsMonitor optionsMonitor, CancellationToken ct) => @@ -358,7 +364,8 @@ internal static class MirrorDomainManagementEndpointExtensions .WithDescription("Generates mirror artifacts from real Concelier advisory data for the selected domain.") .Produces(StatusCodes.Status200OK) .Produces(StatusCodes.Status404NotFound) - .RequireAuthorization(MirrorManagePolicy); + .RequireAuthorization(MirrorManagePolicy) + .Audited("concelier", "execute", resourceType: "mirror_generation"); // GET /domains/{domainId}/status — get domain status group.MapGet("/domains/{domainId}/status", (string domainId, [FromServices] IMirrorDomainStore domainStore) => @@ -400,7 +407,8 @@ internal static class MirrorDomainManagementEndpointExtensions .WithDescription("Imports a mirror bundle from a local filesystem path, persists durable status, and projects the bundle into the live mirror export surface.") .Produces(StatusCodes.Status200OK) .Produces(StatusCodes.Status400BadRequest) - .RequireAuthorization(MirrorManagePolicy); + .RequireAuthorization(MirrorManagePolicy) + .Audited("concelier", "import", resourceType: "mirror_bundle"); // GET /import/status — get status of last import group.MapGet("/import/status", ([FromServices] IMirrorBundleImportStore importStore) => @@ -467,7 +475,8 @@ internal static class MirrorDomainManagementEndpointExtensions .WithSummary("Test mirror consumer endpoint connectivity") .WithDescription("Sends a probe request to the specified mirror base address and reports reachability, latency, and remediation guidance.") .Produces(StatusCodes.Status200OK) - .RequireAuthorization(MirrorManagePolicy); + .RequireAuthorization(MirrorManagePolicy) + .Audited("concelier", "test", resourceType: "mirror_endpoint"); // ===== Consumer connector configuration endpoints ===== @@ -502,7 +511,8 @@ internal static class MirrorDomainManagementEndpointExtensions .WithSummary("Update consumer connector configuration") .WithDescription("Persists mirror consumer settings used by the local backend.") .Produces(StatusCodes.Status200OK) - .RequireAuthorization(MirrorManagePolicy); + .RequireAuthorization(MirrorManagePolicy) + .Audited("concelier", "update", resourceType: "mirror_consumer"); // POST /consumer/discover — fetch mirror index and return available domains group.MapPost("/consumer/discover", async ([FromBody] ConsumerDiscoverRequest request, HttpContext httpContext, CancellationToken ct) => @@ -584,7 +594,8 @@ internal static class MirrorDomainManagementEndpointExtensions .WithDescription("Fetches the mirror index document from the specified base address and returns the list of available domains with metadata including advisory counts, bundle sizes, export formats, and signature status.") .Produces(StatusCodes.Status200OK) .Produces(StatusCodes.Status400BadRequest) - .RequireAuthorization(MirrorManagePolicy); + .RequireAuthorization(MirrorManagePolicy) + .Audited("concelier", "discover", resourceType: "mirror_consumer"); // POST /consumer/verify-signature — fetch bundle header and return signature details group.MapPost("/consumer/verify-signature", async ([FromBody] ConsumerVerifySignatureRequest request, HttpContext httpContext, CancellationToken ct) => @@ -693,7 +704,8 @@ internal static class MirrorDomainManagementEndpointExtensions .WithDescription("Fetches the JWS header from a domain's bundle at the specified mirror and extracts signature metadata including algorithm, key ID, and crypto provider. Only the header is downloaded, not the full bundle.") .Produces(StatusCodes.Status200OK) .Produces(StatusCodes.Status400BadRequest) - .RequireAuthorization(MirrorManagePolicy); + .RequireAuthorization(MirrorManagePolicy) + .Audited("concelier", "verify", resourceType: "mirror_consumer_signature"); } private static MirrorConfigRecord ToMirrorConfigRecord(MirrorConfigOptions options) => new() @@ -1015,10 +1027,10 @@ internal static class MirrorDomainManagementEndpointExtensions }).ToList(); } - private static IReadOnlyList FindUnknownSourceIds(IReadOnlyList sourceIds, IReadOnlyList knownSources) + private static IReadOnlyList FindUnknownSourceIds(IReadOnlyList sourceIds, ISourceRegistry sourceRegistry) { var knownSourceIds = new HashSet( - knownSources.Select(source => source.Key), + sourceRegistry.GetAllSources().Select(source => source.Id), StringComparer.OrdinalIgnoreCase); return sourceIds diff --git a/src/Concelier/StellaOps.Concelier.WebService/Extensions/SbomEndpointExtensions.cs b/src/Concelier/StellaOps.Concelier.WebService/Extensions/SbomEndpointExtensions.cs index 8550f9167..46e67af8d 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Extensions/SbomEndpointExtensions.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Extensions/SbomEndpointExtensions.cs @@ -8,6 +8,7 @@ using HttpResults = Microsoft.AspNetCore.Http.Results; using Microsoft.AspNetCore.Mvc; +using StellaOps.Audit.Emission; using StellaOps.Auth.ServerIntegration.Tenancy; using StellaOps.Concelier.SbomIntegration; using StellaOps.Concelier.SbomIntegration.Models; @@ -63,7 +64,8 @@ internal static class SbomEndpointExtensions .WithDescription("Registers an SBOM by digest, extracts its component PURLs, matches them against the canonical advisory database, and updates the interest score for every matched advisory. Accepts optional reachability and deployment maps to weight reachable/deployed components more heavily.") .Produces(StatusCodes.Status200OK) .Produces(StatusCodes.Status400BadRequest) - .RequireAuthorization("Concelier.Advisories.Ingest"); + .RequireAuthorization("Concelier.Advisories.Ingest") + .Audited("concelier", "create", resourceType: "sbom"); // GET /api/v1/sboms/{digest}/affected - Get advisories affecting an SBOM group.MapGet("/sboms/{digest}/affected", async ( @@ -196,7 +198,8 @@ internal static class SbomEndpointExtensions .WithSummary("Unregister an SBOM") .WithDescription("Removes the SBOM registration identified by digest from the registry, along with all associated PURL-to-canonical match records. Does not modify the interest scores of previously matched advisories.") .Produces(StatusCodes.Status204NoContent) - .RequireAuthorization("Concelier.Advisories.Ingest"); + .RequireAuthorization("Concelier.Advisories.Ingest") + .Audited("concelier", "delete", resourceType: "sbom"); // POST /api/v1/sboms/{digest}/rematch - Rematch SBOM against current advisories group.MapPost("/sboms/{digest}/rematch", async ( @@ -226,7 +229,8 @@ internal static class SbomEndpointExtensions .WithDescription("Re-runs PURL matching for an existing SBOM against the current state of the canonical advisory database and updates match records. Returns the previous and new affected advisory counts so callers can detect newly introduced vulnerabilities.") .Produces(StatusCodes.Status200OK) .Produces(StatusCodes.Status404NotFound) - .RequireAuthorization("Concelier.Advisories.Ingest"); + .RequireAuthorization("Concelier.Advisories.Ingest") + .Audited("concelier", "execute", resourceType: "sbom_match"); // PATCH /api/v1/sboms/{digest} - Incrementally update SBOM (add/remove components) group.MapPatch("/sboms/{digest}", async ( @@ -271,7 +275,8 @@ internal static class SbomEndpointExtensions .WithDescription("Applies an incremental delta to a registered SBOM, adding or removing component PURLs and updating the reachability and deployment maps. After the update, re-runs advisory matching and interest score updates only for the affected components. Supports full replacement mode.") .Produces(StatusCodes.Status200OK) .Produces(StatusCodes.Status404NotFound) - .RequireAuthorization("Concelier.Advisories.Ingest"); + .RequireAuthorization("Concelier.Advisories.Ingest") + .Audited("concelier", "update", resourceType: "sbom"); // GET /api/v1/sboms/stats - Get SBOM registry statistics group.MapGet("/sboms/stats", async ( diff --git a/src/Concelier/StellaOps.Concelier.WebService/Extensions/SourceManagementEndpointExtensions.cs b/src/Concelier/StellaOps.Concelier.WebService/Extensions/SourceManagementEndpointExtensions.cs index 7dd9600f0..34c09222f 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Extensions/SourceManagementEndpointExtensions.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Extensions/SourceManagementEndpointExtensions.cs @@ -29,10 +29,11 @@ internal static class SourceManagementEndpointExtensions // GET /catalog — list all registered source definitions group.MapGet("/catalog", ( - [FromServices] ISourceRegistry registry) => + [FromServices] ISourceRegistry registry, + [FromServices] ConfiguredAdvisorySourceService configuredSources) => { var sources = registry.GetAllSources(); - var items = sources.Select(MapCatalogItem).ToList(); + var items = sources.Select(source => MapCatalogItem(source, configuredSources)).ToList(); return HttpResults.Ok(new SourceCatalogResponse { @@ -46,6 +47,100 @@ internal static class SourceManagementEndpointExtensions .Produces(StatusCodes.Status200OK) .RequireAuthorization(AdvisoryReadPolicy); + group.MapGet("/{sourceId}/configuration", async ( + string sourceId, + [FromServices] ISourceRegistry registry, + [FromServices] ConfiguredAdvisorySourceService configuredSources, + CancellationToken cancellationToken) => + { + var normalizedSourceId = configuredSources.NormalizeSourceId(sourceId); + if (registry.GetSource(normalizedSourceId) is null) + { + return HttpResults.NotFound(new { error = "source_not_found", sourceId = normalizedSourceId }); + } + + if (!configuredSources.SupportsConfiguration(normalizedSourceId)) + { + return HttpResults.UnprocessableEntity(new + { + error = "source_configuration_not_supported", + sourceId = normalizedSourceId, + }); + } + + var snapshot = await configuredSources.GetSourceConfigurationAsync(normalizedSourceId, cancellationToken).ConfigureAwait(false); + if (snapshot is null) + { + return HttpResults.NotFound(new { error = "source_not_found", sourceId = normalizedSourceId }); + } + + return HttpResults.Ok(MapConfiguration(snapshot)); + }) + .WithName("GetSourceConfiguration") + .WithSummary("Get persisted configuration state for a source") + .WithDescription("Returns editable source-specific configuration fields, retained secret state, and current non-secret values for a configured advisory source.") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status404NotFound) + .Produces(StatusCodes.Status422UnprocessableEntity) + .RequireAuthorization(AdvisoryReadPolicy); + + group.MapPut("/{sourceId}/configuration", async ( + string sourceId, + [FromBody] SourceConfigurationUpdateRequest request, + [FromServices] ISourceRegistry registry, + [FromServices] ConfiguredAdvisorySourceService configuredSources, + CancellationToken cancellationToken) => + { + var normalizedSourceId = configuredSources.NormalizeSourceId(sourceId); + if (registry.GetSource(normalizedSourceId) is null) + { + return HttpResults.NotFound(new { error = "source_not_found", sourceId = normalizedSourceId }); + } + + if (!configuredSources.SupportsConfiguration(normalizedSourceId)) + { + return HttpResults.UnprocessableEntity(new + { + error = "source_configuration_not_supported", + sourceId = normalizedSourceId, + }); + } + + try + { + var snapshot = await configuredSources.UpdateSourceConfigurationAsync( + normalizedSourceId, + request.Values, + request.ClearKeys, + cancellationToken).ConfigureAwait(false); + + if (snapshot is null) + { + return HttpResults.NotFound(new { error = "source_not_found", sourceId = normalizedSourceId }); + } + + return HttpResults.Ok(MapConfiguration(snapshot)); + } + catch (InvalidOperationException ex) + { + return HttpResults.BadRequest(new + { + error = "source_configuration_invalid", + sourceId = normalizedSourceId, + message = ex.Message, + }); + } + }) + .WithName("UpdateSourceConfiguration") + .WithSummary("Update persisted configuration for a source") + .WithDescription("Stores source-specific connector configuration, retaining existing secrets when password fields are left blank and clearing only explicit fields requested by the operator.") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status400BadRequest) + .Produces(StatusCodes.Status404NotFound) + .Produces(StatusCodes.Status422UnprocessableEntity) + .RequireAuthorization(SourcesManagePolicy) + .Audited("concelier", "update", resourceType: "advisory_source_configuration"); + // GET /status — enabled sources with last check results group.MapGet("/status", async ( [FromServices] ConfiguredAdvisorySourceService configuredSources, @@ -56,7 +151,9 @@ internal static class SourceManagementEndpointExtensions { SourceId = source.SourceId, Enabled = source.Enabled, - LastCheck = source.LastCheck + LastCheck = source.LastCheck, + SyncSupported = source.SyncSupported, + FetchJobKind = source.FetchJobKind }) .ToList(); @@ -70,21 +167,42 @@ internal static class SourceManagementEndpointExtensions // POST /{sourceId}/enable — enable a single source group.MapPost("/{sourceId}/enable", async ( + HttpContext httpContext, string sourceId, [FromServices] ISourceRegistry registry, [FromServices] ConfiguredAdvisorySourceService configuredSources, CancellationToken cancellationToken) => { - var source = registry.GetSource(sourceId); + var normalizedSourceId = configuredSources.NormalizeSourceId(sourceId); + var source = registry.GetSource(normalizedSourceId); if (source is null) { - return HttpResults.NotFound(new { error = "source_not_found", sourceId }); + return HttpResults.NotFound(new { error = "source_not_found", sourceId = normalizedSourceId }); } - var success = await configuredSources.EnableSourceAsync(sourceId, cancellationToken: cancellationToken).ConfigureAwait(false); + if (!configuredSources.IsSourceRunnable(normalizedSourceId)) + { + return NotImplemented( + httpContext, + detail: $"Source '{normalizedSourceId}' is cataloged but this host does not register a runnable fetch pipeline for it."); + } + + if (await configuredSources.GetConfigurationFailureAsync(normalizedSourceId, cancellationToken).ConfigureAwait(false) is { } configurationFailure) + { + return HttpResults.UnprocessableEntity(new + { + error = "source_config_required", + sourceId = normalizedSourceId, + code = configurationFailure.ErrorCode, + message = configurationFailure.ErrorMessage, + reasons = configurationFailure.PossibleReasons, + }); + } + + var success = await configuredSources.EnableSourceAsync(normalizedSourceId, cancellationToken: cancellationToken).ConfigureAwait(false); return success - ? HttpResults.Ok(new { sourceId, enabled = true }) - : HttpResults.UnprocessableEntity(new { error = "enable_failed", sourceId }); + ? HttpResults.Ok(new { sourceId = normalizedSourceId, enabled = true }) + : HttpResults.UnprocessableEntity(new { error = "enable_failed", sourceId = normalizedSourceId }); }) .WithName("EnableSource") .WithSummary("Enable a source for data ingestion") @@ -102,16 +220,17 @@ internal static class SourceManagementEndpointExtensions [FromServices] ConfiguredAdvisorySourceService configuredSources, CancellationToken cancellationToken) => { - var source = registry.GetSource(sourceId); + var normalizedSourceId = configuredSources.NormalizeSourceId(sourceId); + var source = registry.GetSource(normalizedSourceId); if (source is null) { - return HttpResults.NotFound(new { error = "source_not_found", sourceId }); + return HttpResults.NotFound(new { error = "source_not_found", sourceId = normalizedSourceId }); } - var success = await configuredSources.DisableSourceAsync(sourceId, cancellationToken).ConfigureAwait(false); + var success = await configuredSources.DisableSourceAsync(normalizedSourceId, cancellationToken).ConfigureAwait(false); return success - ? HttpResults.Ok(new { sourceId, enabled = false }) - : HttpResults.UnprocessableEntity(new { error = "disable_failed", sourceId }); + ? HttpResults.Ok(new { sourceId = normalizedSourceId, enabled = false }) + : HttpResults.UnprocessableEntity(new { error = "disable_failed", sourceId = normalizedSourceId }); }) .WithName("DisableSource") .WithSummary("Disable a source") @@ -134,7 +253,8 @@ internal static class SourceManagementEndpointExtensions .WithSummary("Check connectivity for all sources and auto-configure") .WithDescription("Runs connectivity checks against all registered sources. Healthy sources are auto-enabled; failed sources are disabled.") .Produces(StatusCodes.Status200OK) - .RequireAuthorization(SourcesManagePolicy); + .RequireAuthorization(SourcesManagePolicy) + .Audited("concelier", "verify", resourceType: "advisory_source_batch"); // POST /{sourceId}/check — check connectivity for a single source group.MapPost("/{sourceId}/check", async ( @@ -142,10 +262,11 @@ internal static class SourceManagementEndpointExtensions [FromServices] ConfiguredAdvisorySourceService configuredSources, CancellationToken cancellationToken) => { - var result = await configuredSources.CheckSourceConnectivityAsync(sourceId, cancellationToken).ConfigureAwait(false); + var normalizedSourceId = configuredSources.NormalizeSourceId(sourceId); + var result = await configuredSources.CheckSourceConnectivityAsync(normalizedSourceId, cancellationToken).ConfigureAwait(false); if (string.Equals(result.ErrorCode, "SOURCE_NOT_FOUND", StringComparison.OrdinalIgnoreCase)) { - return HttpResults.NotFound(new { error = "source_not_found", sourceId }); + return HttpResults.NotFound(new { error = "source_not_found", sourceId = normalizedSourceId }); } return HttpResults.Ok(result); @@ -155,7 +276,8 @@ internal static class SourceManagementEndpointExtensions .WithDescription("Runs a connectivity check against the specified source and returns detailed status, latency, and remediation steps if failed.") .Produces(StatusCodes.Status200OK) .Produces(StatusCodes.Status404NotFound) - .RequireAuthorization(SourcesManagePolicy); + .RequireAuthorization(SourcesManagePolicy) + .Audited("concelier", "verify", resourceType: "advisory_source"); // POST /batch-enable — enable multiple sources group.MapPost("/batch-enable", async ( @@ -172,15 +294,28 @@ internal static class SourceManagementEndpointExtensions var results = new List(request.SourceIds.Count); foreach (var id in request.SourceIds) { - var source = registry.GetSource(id); + var normalizedSourceId = configuredSources.NormalizeSourceId(id); + var source = registry.GetSource(normalizedSourceId); if (source is null) { - results.Add(new BatchSourceResultItem { SourceId = id, Success = false, Error = "source_not_found" }); + results.Add(new BatchSourceResultItem { SourceId = normalizedSourceId, Success = false, Error = "source_not_found" }); continue; } - var success = await configuredSources.EnableSourceAsync(id, cancellationToken: cancellationToken).ConfigureAwait(false); - results.Add(new BatchSourceResultItem { SourceId = id, Success = success, Error = success ? null : "enable_failed" }); + if (!configuredSources.IsSourceRunnable(normalizedSourceId)) + { + results.Add(new BatchSourceResultItem { SourceId = normalizedSourceId, Success = false, Error = "source_unsupported" }); + continue; + } + + if (await configuredSources.GetConfigurationFailureAsync(normalizedSourceId, cancellationToken).ConfigureAwait(false) is not null) + { + results.Add(new BatchSourceResultItem { SourceId = normalizedSourceId, Success = false, Error = "source_config_required" }); + continue; + } + + var success = await configuredSources.EnableSourceAsync(normalizedSourceId, cancellationToken: cancellationToken).ConfigureAwait(false); + results.Add(new BatchSourceResultItem { SourceId = normalizedSourceId, Success = success, Error = success ? null : "enable_failed" }); } return HttpResults.Ok(new BatchSourceResponse { Results = results }); @@ -208,15 +343,16 @@ internal static class SourceManagementEndpointExtensions var results = new List(request.SourceIds.Count); foreach (var id in request.SourceIds) { - var source = registry.GetSource(id); + var normalizedSourceId = configuredSources.NormalizeSourceId(id); + var source = registry.GetSource(normalizedSourceId); if (source is null) { - results.Add(new BatchSourceResultItem { SourceId = id, Success = false, Error = "source_not_found" }); + results.Add(new BatchSourceResultItem { SourceId = normalizedSourceId, Success = false, Error = "source_not_found" }); continue; } - var success = await configuredSources.DisableSourceAsync(id, cancellationToken).ConfigureAwait(false); - results.Add(new BatchSourceResultItem { SourceId = id, Success = success, Error = success ? null : "disable_failed" }); + var success = await configuredSources.DisableSourceAsync(normalizedSourceId, cancellationToken).ConfigureAwait(false); + results.Add(new BatchSourceResultItem { SourceId = normalizedSourceId, Success = success, Error = success ? null : "disable_failed" }); } return HttpResults.Ok(new BatchSourceResponse { Results = results }); @@ -235,14 +371,35 @@ internal static class SourceManagementEndpointExtensions HttpContext httpContext, string sourceId, [FromServices] ISourceRegistry registry, + [FromServices] ConfiguredAdvisorySourceService configuredSources, [FromServices] IJobCoordinator coordinator, [FromServices] IOptions schedulerOptions, CancellationToken cancellationToken) => { - var source = registry.GetSource(sourceId); + var normalizedSourceId = configuredSources.NormalizeSourceId(sourceId); + var source = registry.GetSource(normalizedSourceId); if (source is null) { - return HttpResults.NotFound(new { error = "source_not_found", sourceId }); + return HttpResults.NotFound(new { error = "source_not_found", sourceId = normalizedSourceId }); + } + + if (!configuredSources.IsSourceRunnable(normalizedSourceId)) + { + return NotImplemented( + httpContext, + $"Source '{normalizedSourceId}' is defined in the catalog but no runnable fetch job is registered in this host."); + } + + if (await configuredSources.GetConfigurationFailureAsync(normalizedSourceId, cancellationToken).ConfigureAwait(false) is { } configurationFailure) + { + return HttpResults.UnprocessableEntity(new + { + error = "source_config_required", + sourceId = normalizedSourceId, + code = configurationFailure.ErrorCode, + message = configurationFailure.ErrorMessage, + reasons = configurationFailure.PossibleReasons, + }); } // Backpressure: reject if active runs are at capacity @@ -254,15 +411,15 @@ internal static class SourceManagementEndpointExtensions return HttpResults.StatusCode(StatusCodes.Status429TooManyRequests); } - var fetchKind = $"source:{sourceId}:fetch"; + var fetchKind = configuredSources.GetFetchJobKind(normalizedSourceId); var result = await coordinator.TriggerAsync(fetchKind, null, "manual", cancellationToken).ConfigureAwait(false); return result.Outcome switch { - JobTriggerOutcome.Accepted => HttpResults.Accepted(null as string, new { sourceId, jobKind = fetchKind, outcome = "accepted", runId = result.Run?.RunId }), - JobTriggerOutcome.AlreadyRunning => HttpResults.Conflict(new { sourceId, jobKind = fetchKind, outcome = "already_running" }), - JobTriggerOutcome.NotFound => HttpResults.Ok(new { sourceId, jobKind = fetchKind, outcome = "no_job_defined", message = $"No fetch job registered for source '{sourceId}'" }), - _ => HttpResults.UnprocessableEntity(new { sourceId, jobKind = fetchKind, outcome = result.Outcome.ToString().ToLowerInvariant(), error = result.ErrorMessage }) + JobTriggerOutcome.Accepted => HttpResults.Accepted(null as string, new { sourceId = normalizedSourceId, jobKind = fetchKind, outcome = "accepted", runId = result.Run?.RunId }), + JobTriggerOutcome.AlreadyRunning => HttpResults.Conflict(new { sourceId = normalizedSourceId, jobKind = fetchKind, outcome = "already_running" }), + JobTriggerOutcome.NotFound => HttpResults.Ok(new { sourceId = normalizedSourceId, jobKind = fetchKind, outcome = "no_job_defined", message = $"No fetch job registered for source '{normalizedSourceId}'" }), + _ => HttpResults.UnprocessableEntity(new { sourceId = normalizedSourceId, jobKind = fetchKind, outcome = result.Outcome.ToString().ToLowerInvariant(), error = result.ErrorMessage }) }; }) .WithName("SyncSource") @@ -284,27 +441,58 @@ internal static class SourceManagementEndpointExtensions CancellationToken cancellationToken) => { var opts = schedulerOptions.Value; - var enabledIds = await configuredSources.GetEnabledSourceIdsAsync(cancellationToken).ConfigureAwait(false); + var enabledIds = await configuredSources.GetPersistedEnabledSourceIdsAsync(cancellationToken).ConfigureAwait(false); var results = new List(enabledIds.Length); + var runnableIds = new List(enabledIds.Length); var batchSize = Math.Max(1, opts.MaxConcurrentJobs); var batchDelay = TimeSpan.FromSeconds(opts.SyncBatchDelaySeconds); - // Trigger in batches to spread load - for (var i = 0; i < enabledIds.Length; i += batchSize) + foreach (var sourceId in enabledIds) { - var batch = enabledIds.AsSpan().Slice(i, Math.Min(batchSize, enabledIds.Length - i)); - var tasks = new List>(batch.Length); - - foreach (var sourceId in batch) + if (!configuredSources.IsSourceRunnable(sourceId)) { - tasks.Add(TriggerSourceAsync(sourceId, coordinator, cancellationToken)); + results.Add(new + { + sourceId, + jobKind = configuredSources.GetFetchJobKind(sourceId), + outcome = "unsupported" + }); + continue; + } + + if (await configuredSources.GetConfigurationFailureAsync(sourceId, cancellationToken).ConfigureAwait(false) is { } configurationFailure) + { + results.Add(new + { + sourceId, + jobKind = configuredSources.GetFetchJobKind(sourceId), + outcome = "config_required", + errorCode = configurationFailure.ErrorCode, + message = configurationFailure.ErrorMessage, + }); + continue; + } + + runnableIds.Add(sourceId); + } + + // Trigger in batches to spread load + for (var i = 0; i < runnableIds.Count; i += batchSize) + { + var batchCount = Math.Min(batchSize, runnableIds.Count - i); + var tasks = new List>(batchCount); + + for (var offset = 0; offset < batchCount; offset++) + { + var sourceId = runnableIds[i + offset]; + tasks.Add(TriggerSourceAsync(sourceId, configuredSources, coordinator, cancellationToken)); } var batchResults = await Task.WhenAll(tasks).ConfigureAwait(false); results.AddRange(batchResults); // Delay between batches (skip after last batch) - if (i + batchSize < enabledIds.Length) + if (i + batchSize < runnableIds.Count) { await Task.Delay(batchDelay, cancellationToken).ConfigureAwait(false); } @@ -317,7 +505,8 @@ internal static class SourceManagementEndpointExtensions .WithSummary("Trigger data sync for all enabled advisory sources") .WithDescription("Triggers fetch jobs for all enabled sources in batches to prevent resource exhaustion. Batches are sized by MaxConcurrentJobs with a configurable inter-batch delay.") .Produces(StatusCodes.Status200OK) - .RequireAuthorization(SourcesManagePolicy); + .RequireAuthorization(SourcesManagePolicy) + .Audited("concelier", "execute", resourceType: "advisory_source_sync_batch"); // GET /{sourceId}/check-result — get last check result for a source group.MapGet("/{sourceId}/check-result", ( @@ -346,7 +535,7 @@ internal static class SourceManagementEndpointExtensions .RequireAuthorization(AdvisoryReadPolicy); } - private static SourceCatalogItem MapCatalogItem(SourceDefinition source) + private static SourceCatalogItem MapCatalogItem(SourceDefinition source, ConfiguredAdvisorySourceService configuredSources) { return new SourceCatalogItem { @@ -363,13 +552,42 @@ internal static class SourceManagementEndpointExtensions DefaultPriority = source.DefaultPriority, Regions = source.Regions, Tags = source.Tags, - EnabledByDefault = source.EnabledByDefault + EnabledByDefault = source.EnabledByDefault, + SupportsConfiguration = configuredSources.SupportsConfiguration(source.Id), + SyncSupported = configuredSources.IsSourceRunnable(source.Id), + FetchJobKind = configuredSources.GetFetchJobKind(source.Id) }; } - private static async Task TriggerSourceAsync(string sourceId, IJobCoordinator coordinator, CancellationToken cancellationToken) + private static SourceConfigurationResponse MapConfiguration(AdvisorySourceConfigurationSnapshot snapshot) + => new() + { + SourceId = snapshot.SourceId, + DisplayName = snapshot.DisplayName, + Fields = snapshot.Fields + .Select(field => new SourceConfigurationFieldItem + { + Key = field.Key, + Label = field.Label, + InputType = field.InputType, + Sensitive = field.Sensitive, + Required = field.Required, + Value = field.Value, + HasValue = field.HasValue, + IsSecretRetained = field.IsSecretRetained, + HelpText = field.HelpText, + Placeholder = field.Placeholder, + }) + .ToList(), + }; + + private static async Task TriggerSourceAsync( + string sourceId, + ConfiguredAdvisorySourceService configuredSources, + IJobCoordinator coordinator, + CancellationToken cancellationToken) { - var fetchKind = $"source:{sourceId}:fetch"; + var fetchKind = configuredSources.GetFetchJobKind(sourceId); var result = await coordinator.TriggerAsync(fetchKind, null, "manual-all", cancellationToken).ConfigureAwait(false); return new { @@ -414,6 +632,36 @@ public sealed record SourceCatalogItem public IReadOnlyList Regions { get; init; } = []; public IReadOnlyList Tags { get; init; } = []; public bool EnabledByDefault { get; init; } + public bool SupportsConfiguration { get; init; } + public bool SyncSupported { get; init; } + public string FetchJobKind { get; init; } = string.Empty; +} + +public sealed record SourceConfigurationResponse +{ + public string SourceId { get; init; } = string.Empty; + public string DisplayName { get; init; } = string.Empty; + public IReadOnlyList Fields { get; init; } = []; +} + +public sealed record SourceConfigurationFieldItem +{ + public string Key { get; init; } = string.Empty; + public string Label { get; init; } = string.Empty; + public string InputType { get; init; } = string.Empty; + public bool Sensitive { get; init; } + public bool Required { get; init; } + public string? Value { get; init; } + public bool HasValue { get; init; } + public bool IsSecretRetained { get; init; } + public string? HelpText { get; init; } + public string? Placeholder { get; init; } +} + +public sealed record SourceConfigurationUpdateRequest +{ + public IReadOnlyDictionary? Values { get; init; } + public IReadOnlyList? ClearKeys { get; init; } } public sealed record SourceStatusResponse @@ -426,6 +674,8 @@ public sealed record SourceStatusItem public string SourceId { get; init; } = string.Empty; public bool Enabled { get; init; } public SourceConnectivityResult? LastCheck { get; init; } + public bool SyncSupported { get; init; } + public string FetchJobKind { get; init; } = string.Empty; } public sealed record BatchSourceRequest diff --git a/src/Concelier/StellaOps.Concelier.WebService/Extensions/TopologySetupEndpointExtensions.cs b/src/Concelier/StellaOps.Concelier.WebService/Extensions/TopologySetupEndpointExtensions.cs index 401671104..ebb9392a2 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Extensions/TopologySetupEndpointExtensions.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Extensions/TopologySetupEndpointExtensions.cs @@ -1,5 +1,6 @@ using HttpResults = Microsoft.AspNetCore.Http.Results; using Microsoft.AspNetCore.Mvc; +using StellaOps.Audit.Emission; using StellaOps.Concelier.WebService.Services; using StellaOps.ReleaseOrchestrator.Environment.Agent; using StellaOps.ReleaseOrchestrator.Environment.Deletion; @@ -77,7 +78,8 @@ internal static class TopologySetupEndpointExtensions }) .RequireAuthorization(TopologyManagePolicy) .WithName("CreateEnvironment") - .WithSummary("Create a new environment in the topology"); + .WithSummary("Create a new environment in the topology") + .Audited("concelier", "create", resourceType: "environment"); group.MapGet("/", async ( [FromServices] IEnvironmentService environmentService, @@ -135,7 +137,8 @@ internal static class TopologySetupEndpointExtensions }) .RequireAuthorization(TopologyManagePolicy) .WithName("CreateTarget") - .WithSummary("Create a new deployment target"); + .WithSummary("Create a new deployment target") + .Audited("concelier", "create", resourceType: "target"); group.MapGet("/", async ( [FromQuery] Guid? environmentId, @@ -165,7 +168,8 @@ internal static class TopologySetupEndpointExtensions }) .RequireAuthorization(TopologyManagePolicy) .WithName("AssignAgent") - .WithSummary("Assign an agent to a target"); + .WithSummary("Assign an agent to a target") + .Audited("concelier", "update", resourceType: "target"); } // ── Agent List (for topology wizard) ────────────────────── @@ -236,7 +240,8 @@ internal static class TopologySetupEndpointExtensions .WithName("CreateRegion") .WithSummary("Create a new region") .Produces(StatusCodes.Status201Created) - .RequireAuthorization(TopologyManagePolicy); + .RequireAuthorization(TopologyManagePolicy) + .Audited("concelier", "create", resourceType: "region"); group.MapGet("/", async ( [FromServices] IRegionService regionService, @@ -283,7 +288,8 @@ internal static class TopologySetupEndpointExtensions .WithName("UpdateRegion") .WithSummary("Update a region") .Produces(StatusCodes.Status200OK) - .RequireAuthorization(TopologyManagePolicy); + .RequireAuthorization(TopologyManagePolicy) + .Audited("concelier", "update", resourceType: "region"); group.MapDelete("/{id:guid}", async ( Guid id, @@ -296,7 +302,8 @@ internal static class TopologySetupEndpointExtensions .WithName("DeleteRegion") .WithSummary("Delete a region") .Produces(StatusCodes.Status204NoContent) - .RequireAuthorization(TopologyAdminPolicy); + .RequireAuthorization(TopologyAdminPolicy) + .Audited("concelier", "delete", resourceType: "region"); } // ── Infrastructure Binding Endpoints ───────────────────────── @@ -320,7 +327,8 @@ internal static class TopologySetupEndpointExtensions .WithName("CreateInfrastructureBinding") .WithSummary("Bind an integration to a scope") .Produces(StatusCodes.Status201Created) - .RequireAuthorization(TopologyManagePolicy); + .RequireAuthorization(TopologyManagePolicy) + .Audited("concelier", "create", resourceType: "infrastructure_binding"); group.MapDelete("/{id:guid}", async ( Guid id, @@ -333,7 +341,8 @@ internal static class TopologySetupEndpointExtensions .WithName("DeleteInfrastructureBinding") .WithSummary("Remove an infrastructure binding") .Produces(StatusCodes.Status204NoContent) - .RequireAuthorization(TopologyManagePolicy); + .RequireAuthorization(TopologyManagePolicy) + .Audited("concelier", "delete", resourceType: "infrastructure_binding"); group.MapGet("/", async ( [FromQuery] string scopeType, @@ -396,7 +405,8 @@ internal static class TopologySetupEndpointExtensions .WithName("TestInfrastructureBinding") .WithSummary("Test binding connectivity") .Produces(StatusCodes.Status200OK) - .RequireAuthorization(TopologyManagePolicy); + .RequireAuthorization(TopologyManagePolicy) + .Audited("concelier", "test", resourceType: "infrastructure_binding"); } // ── Readiness Endpoints ────────────────────────────────────── @@ -416,7 +426,8 @@ internal static class TopologySetupEndpointExtensions }) .WithName("ValidateTarget") .WithSummary("Run all readiness gates for a target") - .Produces(StatusCodes.Status200OK); + .Produces(StatusCodes.Status200OK) + .Audited("concelier", "verify", resourceType: "target_readiness"); targets.MapGet("/{id:guid}/readiness", async ( Guid id, @@ -493,6 +504,7 @@ internal static class TopologySetupEndpointExtensions foreach (var (path, entityType) in entityTypes) { + var resourceType = entityType.ToString().ToLowerInvariant(); app.MapPatch($"/api/v1/{path}/{{id:guid}}/name", async ( Guid id, [FromBody] RenameApiRequest body, @@ -515,7 +527,8 @@ internal static class TopologySetupEndpointExtensions .WithTags("Topology Rename") .Produces(StatusCodes.Status200OK) .Produces(StatusCodes.Status409Conflict) - .RequireAuthorization(TopologyManagePolicy); + .RequireAuthorization(TopologyManagePolicy) + .Audited("concelier", "update", resourceType: resourceType); } } @@ -535,6 +548,7 @@ internal static class TopologySetupEndpointExtensions // Request deletion for each entity type foreach (var (path, entityType) in entityPaths) { + var resourceType = entityType.ToString().ToLowerInvariant(); app.MapPost($"/api/v1/{path}/{{id:guid}}/request-delete", async ( Guid id, [FromBody] RequestDeleteApiRequest? body, @@ -549,7 +563,8 @@ internal static class TopologySetupEndpointExtensions .WithSummary($"Request deletion of a {entityType.ToString().ToLowerInvariant()} with cool-off") .WithTags("Topology Deletion") .Produces(StatusCodes.Status202Accepted) - .RequireAuthorization(TopologyManagePolicy); + .RequireAuthorization(TopologyManagePolicy) + .Audited("concelier", "request_delete", resourceType: resourceType); } // Pending deletion management @@ -568,7 +583,8 @@ internal static class TopologySetupEndpointExtensions .WithName("ConfirmDeletion") .WithSummary("Confirm a pending deletion after cool-off expires") .Produces(StatusCodes.Status200OK) - .RequireAuthorization(TopologyAdminPolicy); + .RequireAuthorization(TopologyAdminPolicy) + .Audited("concelier", "delete", resourceType: "pending_deletion"); deletionGroup.MapPost("/{id:guid}/cancel", async ( Guid id, @@ -582,7 +598,8 @@ internal static class TopologySetupEndpointExtensions .WithName("CancelDeletion") .WithSummary("Cancel a pending deletion") .Produces(StatusCodes.Status204NoContent) - .RequireAuthorization(TopologyManagePolicy); + .RequireAuthorization(TopologyManagePolicy) + .Audited("concelier", "cancel_delete", resourceType: "pending_deletion"); deletionGroup.MapGet("/", async ( [FromServices] IPendingDeletionService deletionService, diff --git a/src/Concelier/StellaOps.Concelier.WebService/Program.cs b/src/Concelier/StellaOps.Concelier.WebService/Program.cs index 2deb8f258..36bda3500 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Program.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Program.cs @@ -37,7 +37,13 @@ using StellaOps.Concelier.Core.Orchestration; using StellaOps.Concelier.Core.Raw; using StellaOps.Concelier.Core.Signals; using StellaOps.Concelier.Core.Sources; +using StellaOps.Concelier.Connector.Ghsa.Configuration; using StellaOps.Concelier.Connector.StellaOpsMirror.Settings; +using StellaOps.Concelier.Connector.Vndr.Adobe.Configuration; +using StellaOps.Concelier.Connector.Vndr.Cisco.Configuration; +using StellaOps.Concelier.Connector.Vndr.Chromium.Configuration; +using StellaOps.Concelier.Connector.Vndr.Msrc.Configuration; +using StellaOps.Concelier.Connector.Vndr.Oracle.Configuration; using StellaOps.Concelier.Merge; using StellaOps.Concelier.Merge.Services; using StellaOps.Concelier.Models; @@ -428,9 +434,6 @@ if (builder.Environment.IsEnvironment("Testing")) concelierOptions.Authority.ClientScopes.Add(StellaOpsScopes.ConcelierJobsTrigger); } - // Register in-memory storage stubs for Testing to satisfy merge module dependencies - builder.Services.AddInMemoryStorage(); - // Skip validation in Testing to allow factory-provided wiring. } else @@ -515,7 +518,7 @@ builder.Services.AddOptions() { options.Subject ??= "concelier.advisory.observation.updated.v1"; options.Stream ??= "CONCELIER_OBS"; - options.Transport = string.IsNullOrWhiteSpace(options.Transport) ? "inmemory" : options.Transport; + options.Transport = string.IsNullOrWhiteSpace(options.Transport) ? "postgres" : options.Transport; }) .ValidateOnStart(); builder.Services.AddConcelierAocGuards(); @@ -565,6 +568,16 @@ builder.Services.AddConcelierFederationServices(); // Register advisory source registry and connectivity services builder.Services.AddSourcesRegistry(builder.Configuration); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddHostedService(); +builder.Services.AddSingleton>(sp => sp.GetRequiredService()); +builder.Services.AddSingleton>(sp => sp.GetRequiredService()); +builder.Services.AddSingleton>(sp => sp.GetRequiredService()); +builder.Services.AddSingleton>(sp => sp.GetRequiredService()); +builder.Services.AddSingleton>(sp => sp.GetRequiredService()); +builder.Services.AddSingleton>(sp => sp.GetRequiredService()); builder.Services.AddScoped(); // Mirror domain management is backed by Concelier PostgreSQL state so the @@ -1249,11 +1262,6 @@ orchestratorGroup.MapPost("/registry", async ( return tenantError; } - if (store is UnsupportedOrchestratorRegistryStore) - { - return RuntimeNotImplemented(context, UnsupportedOrchestratorRegistryStore.Detail); - } - if (string.IsNullOrWhiteSpace(request.ConnectorId) || string.IsNullOrWhiteSpace(request.Source)) { return Problem(context, "connectorId and source are required", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide connectorId and source."); @@ -1281,7 +1289,8 @@ orchestratorGroup.MapPost("/registry", async ( await store.UpsertAsync(record, cancellationToken).ConfigureAwait(false); return HttpResults.Accepted(); -}).WithName("UpsertOrchestratorRegistry"); +}).WithName("UpsertOrchestratorRegistry") + .Audited("concelier", "update", resourceType: "orchestrator_registry"); orchestratorGroup.MapPost("/heartbeat", async ( HttpContext context, @@ -1295,11 +1304,6 @@ orchestratorGroup.MapPost("/heartbeat", async ( return tenantError; } - if (store is UnsupportedOrchestratorRegistryStore) - { - return RuntimeNotImplemented(context, UnsupportedOrchestratorRegistryStore.Detail); - } - if (string.IsNullOrWhiteSpace(request.ConnectorId)) { return Problem(context, "connectorId is required", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide connectorId."); @@ -1341,11 +1345,6 @@ orchestratorGroup.MapPost("/commands", async ( return tenantError; } - if (store is UnsupportedOrchestratorRegistryStore) - { - return RuntimeNotImplemented(context, UnsupportedOrchestratorRegistryStore.Detail); - } - if (string.IsNullOrWhiteSpace(request.ConnectorId)) { return Problem(context, "connectorId is required", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide connectorId."); @@ -1375,9 +1374,10 @@ orchestratorGroup.MapPost("/commands", async ( timeProvider.GetUtcNow(), request.ExpiresAt); - await store.EnqueueCommandAsync(command, cancellationToken).ConfigureAwait(false); + await store.EnqueueCommandAsync(command, cancellationToken).ConfigureAwait(false); return HttpResults.Accepted(); -}).WithName("EnqueueOrchestratorCommand"); +}).WithName("EnqueueOrchestratorCommand") + .Audited("concelier", "execute", resourceType: "orchestrator_command"); orchestratorGroup.MapGet("/commands", async ( HttpContext context, @@ -1394,11 +1394,6 @@ orchestratorGroup.MapGet("/commands", async ( return tenantError; } - if (store is UnsupportedOrchestratorRegistryStore) - { - return RuntimeNotImplemented(context, UnsupportedOrchestratorRegistryStore.Detail); - } - if (string.IsNullOrWhiteSpace(connectorId)) { return Problem(context, "connectorId is required", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide connectorId."); @@ -1854,6 +1849,7 @@ if (authorityConfigured) { advisoryIngestEndpoint.RequireAuthorization(AdvisoryIngestPolicyName); } +advisoryIngestEndpoint.Audited("concelier", "create", resourceType: "advisory_raw"); var advisoryRawListEndpoint = app.MapGet("/advisories/raw", async ( HttpContext context, @@ -2359,7 +2355,8 @@ app.MapPost("/internal/events/observations/publish", async ( } return HttpResults.Ok(new { tenant, published, requestedCount = request.ObservationIds.Count, timestamp = timeProvider.GetUtcNow() }); -}).WithName("PublishObservationEvents"); +}).WithName("PublishObservationEvents") + .Audited("concelier", "publish", resourceType: "advisory_observation_event"); // Internal endpoint for publishing linkset events to NATS/Redis. // Publishes advisory.linkset.updated@1 events with idempotent keys and tenant + provenance references. @@ -2406,7 +2403,8 @@ app.MapPost("/internal/events/linksets/publish", async ( } return HttpResults.Ok(new { tenant, published, requestedCount = request.AdvisoryIds.Count, hasMore = result.HasMore, timestamp = timeProvider.GetUtcNow() }); -}).WithName("PublishLinksetEvents"); +}).WithName("PublishLinksetEvents") + .Audited("concelier", "publish", resourceType: "advisory_linkset_event"); var advisoryEvidenceEndpoint = app.MapGet("/vuln/evidence/advisories/{advisoryKey}", async ( string advisoryKey, @@ -2517,6 +2515,7 @@ if (authorityConfigured) { attestationVerifyEndpoint.RequireAuthorization(AdvisoryReadPolicyName); } +attestationVerifyEndpoint.Audited("concelier", "verify", resourceType: "evidence_attestation"); // Evidence snapshot (manifest-only) endpoint for Console/VEX consumers var evidenceSnapshotEndpoint = app.MapGet("/obs/evidence/advisories/{advisoryKey}", async ( @@ -2697,6 +2696,7 @@ if (authorityConfigured) { incidentUpsertEndpoint.RequireAuthorization(AdvisoryReadPolicyName); } +incidentUpsertEndpoint.Audited("concelier", "update", resourceType: "advisory_incident"); var incidentDeleteEndpoint = app.MapDelete("/obs/incidents/advisories/{advisoryKey}", async ( string advisoryKey, @@ -2719,6 +2719,7 @@ if (authorityConfigured) { incidentDeleteEndpoint.RequireAuthorization(AdvisoryReadPolicyName); } +incidentDeleteEndpoint.Audited("concelier", "delete", resourceType: "advisory_incident"); var advisoryChunksEndpoint = app.MapGet("/advisories/{advisoryKey}/chunks", async ( string advisoryKey, @@ -3111,6 +3112,7 @@ if (authorityConfigured) { aocVerifyEndpoint.RequireAuthorization(AocVerifyPolicyName); } +aocVerifyEndpoint.Audited("concelier", "verify", resourceType: "aoc_window"); app.MapGet("/concelier/advisories/{vulnerabilityKey}/replay", async ( string vulnerabilityKey, @@ -3206,6 +3208,7 @@ if (authorityConfigured) { statementProvenanceEndpoint.RequireAuthorization(AdvisoryIngestPolicyName); } +statementProvenanceEndpoint.Audited("concelier", "create", resourceType: "statement_provenance"); var loggingEnabled = concelierOptions.Telemetry?.EnableLogging ?? true; @@ -4167,11 +4170,6 @@ var jobsListEndpoint = app.MapGet("/jobs", async (string? kind, int? limit, [Fro { ApplyNoCache(context.Response); - if (coordinator is UnsupportedJobCoordinator) - { - return RuntimeNotImplemented(context, UnsupportedJobCoordinator.Detail); - } - var take = Math.Clamp(limit.GetValueOrDefault(50), 1, 200); var runs = await coordinator.GetRecentRunsAsync(kind, take, cancellationToken).ConfigureAwait(false); var payload = runs.Select(JobRunResponse.FromSnapshot).ToArray(); @@ -4186,11 +4184,6 @@ var jobByIdEndpoint = app.MapGet("/jobs/{runId:guid}", async (Guid runId, [FromS { ApplyNoCache(context.Response); - if (coordinator is UnsupportedJobCoordinator) - { - return RuntimeNotImplemented(context, UnsupportedJobCoordinator.Detail); - } - var run = await coordinator.GetRunAsync(runId, cancellationToken).ConfigureAwait(false); if (run is null) { @@ -4208,11 +4201,6 @@ var jobDefinitionsEndpoint = app.MapGet("/jobs/definitions", async ([FromService { ApplyNoCache(context.Response); - if (coordinator is UnsupportedJobCoordinator) - { - return RuntimeNotImplemented(context, UnsupportedJobCoordinator.Detail); - } - var definitions = await coordinator.GetDefinitionsAsync(cancellationToken).ConfigureAwait(false); if (definitions.Count == 0) { @@ -4240,11 +4228,6 @@ var jobDefinitionEndpoint = app.MapGet("/jobs/definitions/{kind}", async (string { ApplyNoCache(context.Response); - if (coordinator is UnsupportedJobCoordinator) - { - return RuntimeNotImplemented(context, UnsupportedJobCoordinator.Detail); - } - var definition = (await coordinator.GetDefinitionsAsync(cancellationToken).ConfigureAwait(false)) .FirstOrDefault(d => string.Equals(d.Kind, kind, StringComparison.Ordinal)); @@ -4268,11 +4251,6 @@ var jobDefinitionRunsEndpoint = app.MapGet("/jobs/definitions/{kind}/runs", asyn { ApplyNoCache(context.Response); - if (coordinator is UnsupportedJobCoordinator) - { - return RuntimeNotImplemented(context, UnsupportedJobCoordinator.Detail); - } - var definition = (await coordinator.GetDefinitionsAsync(cancellationToken).ConfigureAwait(false)) .FirstOrDefault(d => string.Equals(d.Kind, kind, StringComparison.Ordinal)); @@ -4295,11 +4273,6 @@ var activeJobsEndpoint = app.MapGet("/jobs/active", async ([FromServices] IJobCo { ApplyNoCache(context.Response); - if (coordinator is UnsupportedJobCoordinator) - { - return RuntimeNotImplemented(context, UnsupportedJobCoordinator.Detail); - } - var runs = await coordinator.GetActiveRunsAsync(cancellationToken).ConfigureAwait(false); var payload = runs.Select(JobRunResponse.FromSnapshot).ToArray(); return JsonResult(payload); @@ -4313,11 +4286,6 @@ var triggerJobEndpoint = app.MapPost("/jobs/{*jobKind}", async (string jobKind, { ApplyNoCache(context.Response); - if (coordinator is UnsupportedJobCoordinator) - { - return RuntimeNotImplemented(context, UnsupportedJobCoordinator.Detail); - } - request ??= new JobTriggerRequest(); request.Parameters ??= new Dictionary(StringComparer.Ordinal); var trigger = string.IsNullOrWhiteSpace(request.Trigger) ? "api" : request.Trigger; @@ -4398,6 +4366,7 @@ if (enforceAuthority) { triggerJobEndpoint.RequireAuthorization(JobsPolicyName); } +triggerJobEndpoint.Audited("concelier", "execute", resourceType: "job"); var concelierHealthEndpoint = app.MapGet("/obs/concelier/health", ( HttpContext context, diff --git a/src/Concelier/StellaOps.Concelier.WebService/Services/AdvisorySourceConfigurationModels.cs b/src/Concelier/StellaOps.Concelier.WebService/Services/AdvisorySourceConfigurationModels.cs new file mode 100644 index 000000000..2284c259c --- /dev/null +++ b/src/Concelier/StellaOps.Concelier.WebService/Services/AdvisorySourceConfigurationModels.cs @@ -0,0 +1,222 @@ +using System.Collections.Immutable; +using StellaOps.Concelier.Core.Sources; + +namespace StellaOps.Concelier.WebService.Services; + +internal sealed record AdvisorySourceConfigurationSchema( + string SourceId, + ImmutableArray Fields); + +internal sealed record AdvisorySourceConfigurationFieldDefinition +{ + public AdvisorySourceConfigurationFieldDefinition( + string key, + string label, + string inputType, + bool sensitive, + bool required, + string? helpText = null, + string? placeholder = null, + ImmutableArray aliases = default) + { + Key = key; + Label = label; + InputType = inputType; + Sensitive = sensitive; + Required = required; + HelpText = helpText; + Placeholder = placeholder; + Aliases = aliases.IsDefault ? [] : aliases; + } + + public string Key { get; init; } + public string Label { get; init; } + public string InputType { get; init; } + public bool Sensitive { get; init; } + public bool Required { get; init; } + public string? HelpText { get; init; } + public string? Placeholder { get; init; } + public ImmutableArray Aliases { get; init; } +} + +public sealed record AdvisorySourceConfigurationSnapshot( + string SourceId, + string DisplayName, + ImmutableArray Fields); + +public sealed record AdvisorySourceConfigurationFieldState( + string Key, + string Label, + string InputType, + bool Sensitive, + bool Required, + string? Value, + bool HasValue, + bool IsSecretRetained, + string? HelpText, + string? Placeholder); + +internal static class AdvisorySourceConfigurationDefinitions +{ + private static readonly ImmutableDictionary Schemas = + ImmutableDictionary.CreateRange(StringComparer.OrdinalIgnoreCase, + [ + CreateGhsa(), + CreateCisco(), + CreateMicrosoft(), + CreateOracle(), + CreateAdobe(), + CreateChromium(), + ]); + + public static AdvisorySourceConfigurationSchema? Get(string sourceId) + => Schemas.TryGetValue(SourceKeyAliases.Normalize(sourceId), out var schema) + ? schema + : null; + + public static IReadOnlyCollection ConfigurableSourceIds => Schemas.Keys.ToImmutableArray(); + + private static KeyValuePair CreateGhsa() + { + var schema = new AdvisorySourceConfigurationSchema( + "ghsa", + [ + new AdvisorySourceConfigurationFieldDefinition( + key: "apiToken", + label: "API Token", + inputType: "password", + sensitive: true, + required: true, + helpText: "GitHub Personal Access Token or GitHub App token with access to Security Advisory REST APIs.", + placeholder: "ghp_xxx or github_pat_xxx", + aliases: ["token"]), + ]); + + return new KeyValuePair(schema.SourceId, schema); + } + + private static KeyValuePair CreateCisco() + { + var schema = new AdvisorySourceConfigurationSchema( + "cisco", + [ + new AdvisorySourceConfigurationFieldDefinition( + key: "clientId", + label: "OAuth Client ID", + inputType: "text", + sensitive: false, + required: true, + helpText: "Client ID for the Cisco PSIRT / OpenVuln OAuth application."), + new AdvisorySourceConfigurationFieldDefinition( + key: "clientSecret", + label: "OAuth Client Secret", + inputType: "password", + sensitive: true, + required: true, + helpText: "Client secret paired with the Cisco PSIRT / OpenVuln OAuth client."), + ]); + + return new KeyValuePair(schema.SourceId, schema); + } + + private static KeyValuePair CreateMicrosoft() + { + var schema = new AdvisorySourceConfigurationSchema( + "microsoft", + [ + new AdvisorySourceConfigurationFieldDefinition( + key: "tenantId", + label: "Azure Tenant ID", + inputType: "text", + sensitive: false, + required: true, + helpText: "Microsoft Entra tenant GUID used for the client-credential flow."), + new AdvisorySourceConfigurationFieldDefinition( + key: "clientId", + label: "Azure Application (Client) ID", + inputType: "text", + sensitive: false, + required: true, + helpText: "Application (client) ID of the MSRC-integrated app registration."), + new AdvisorySourceConfigurationFieldDefinition( + key: "clientSecret", + label: "Azure Client Secret", + inputType: "password", + sensitive: true, + required: true, + helpText: "Client secret created for the MSRC-integrated app registration."), + ]); + + return new KeyValuePair(schema.SourceId, schema); + } + + private static KeyValuePair CreateOracle() + { + var schema = new AdvisorySourceConfigurationSchema( + "oracle", + [ + new AdvisorySourceConfigurationFieldDefinition( + key: "calendarUris", + label: "Calendar / Index URIs", + inputType: "textarea", + sensitive: false, + required: false, + helpText: "One absolute URI per line. Leave blank to keep the Oracle security alerts landing page default.", + placeholder: "https://www.oracle.com/security-alerts/"), + new AdvisorySourceConfigurationFieldDefinition( + key: "advisoryUris", + label: "Pinned Advisory URIs", + inputType: "textarea", + sensitive: false, + required: false, + helpText: "Optional absolute Oracle advisory URIs to fetch directly, one per line.", + placeholder: "https://www.oracle.com/security-alerts/cpujan2026.html"), + ]); + + return new KeyValuePair(schema.SourceId, schema); + } + + private static KeyValuePair CreateAdobe() + { + var schema = new AdvisorySourceConfigurationSchema( + "adobe", + [ + new AdvisorySourceConfigurationFieldDefinition( + key: "indexUri", + label: "Primary Index URI", + inputType: "text", + sensitive: false, + required: false, + helpText: "Leave blank to keep the canonical Adobe bulletin index default.", + placeholder: "https://helpx.adobe.com/security/security-bulletin.html"), + new AdvisorySourceConfigurationFieldDefinition( + key: "additionalIndexUris", + label: "Additional Index URIs", + inputType: "textarea", + sensitive: false, + required: false, + helpText: "Optional absolute Adobe mirror or index URIs, one per line.", + placeholder: "https://mirror.example.internal/adobe/security-bulletin.html"), + ]); + + return new KeyValuePair(schema.SourceId, schema); + } + + private static KeyValuePair CreateChromium() + { + var schema = new AdvisorySourceConfigurationSchema( + "chromium", + [ + new AdvisorySourceConfigurationFieldDefinition( + key: "feedUri", + label: "Feed URI", + inputType: "text", + sensitive: false, + required: false, + helpText: "Leave blank to keep the canonical Chrome Releases Atom feed default.", + placeholder: "https://chromereleases.googleblog.com/atom.xml"), + ]); + + return new KeyValuePair(schema.SourceId, schema); + } +} diff --git a/src/Concelier/StellaOps.Concelier.WebService/Services/AdvisorySourceRuntimeOptionsInvalidator.cs b/src/Concelier/StellaOps.Concelier.WebService/Services/AdvisorySourceRuntimeOptionsInvalidator.cs new file mode 100644 index 000000000..e7eba7040 --- /dev/null +++ b/src/Concelier/StellaOps.Concelier.WebService/Services/AdvisorySourceRuntimeOptionsInvalidator.cs @@ -0,0 +1,74 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Ghsa.Configuration; +using StellaOps.Concelier.Connector.Vndr.Cisco.Configuration; +using StellaOps.Concelier.Connector.Vndr.Cisco.Internal; +using StellaOps.Concelier.Connector.Vndr.Msrc.Configuration; +using StellaOps.Concelier.Connector.Vndr.Msrc.Internal; +using StellaOps.Concelier.Connector.Vndr.Oracle.Configuration; +using StellaOps.Concelier.Core.Sources; + +namespace StellaOps.Concelier.WebService.Services; + +public sealed class AdvisorySourceRuntimeOptionsInvalidator +{ + private readonly IOptionsMonitorCache ghsaOptionsCache; + private readonly IOptionsMonitorCache ciscoOptionsCache; + private readonly IOptionsMonitorCache msrcOptionsCache; + private readonly IOptionsMonitorCache oracleOptionsCache; + private readonly IOptionsMonitorCache sourceHttpClientOptionsCache; + private readonly IServiceProvider serviceProvider; + + public AdvisorySourceRuntimeOptionsInvalidator( + IOptionsMonitorCache ghsaOptionsCache, + IOptionsMonitorCache ciscoOptionsCache, + IOptionsMonitorCache msrcOptionsCache, + IOptionsMonitorCache oracleOptionsCache, + IOptionsMonitorCache sourceHttpClientOptionsCache, + IServiceProvider serviceProvider) + { + this.ghsaOptionsCache = ghsaOptionsCache ?? throw new ArgumentNullException(nameof(ghsaOptionsCache)); + this.ciscoOptionsCache = ciscoOptionsCache ?? throw new ArgumentNullException(nameof(ciscoOptionsCache)); + this.msrcOptionsCache = msrcOptionsCache ?? throw new ArgumentNullException(nameof(msrcOptionsCache)); + this.oracleOptionsCache = oracleOptionsCache ?? throw new ArgumentNullException(nameof(oracleOptionsCache)); + this.sourceHttpClientOptionsCache = sourceHttpClientOptionsCache ?? throw new ArgumentNullException(nameof(sourceHttpClientOptionsCache)); + this.serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); + } + + public void Invalidate(string sourceId) + { + var normalizedSourceId = SourceKeyAliases.Normalize(sourceId); + switch (normalizedSourceId) + { + case "ghsa": + ghsaOptionsCache.TryRemove(Microsoft.Extensions.Options.Options.DefaultName); + sourceHttpClientOptionsCache.TryRemove(GhsaOptions.HttpClientName); + break; + case "cisco": + ciscoOptionsCache.TryRemove(Microsoft.Extensions.Options.Options.DefaultName); + sourceHttpClientOptionsCache.TryRemove(CiscoOptions.HttpClientName); + sourceHttpClientOptionsCache.TryRemove(CiscoOptions.AuthHttpClientName); + serviceProvider.GetService()?.Invalidate(); + break; + case "microsoft": + msrcOptionsCache.TryRemove(Microsoft.Extensions.Options.Options.DefaultName); + sourceHttpClientOptionsCache.TryRemove(MsrcOptions.HttpClientName); + sourceHttpClientOptionsCache.TryRemove(MsrcOptions.TokenClientName); + serviceProvider.GetService()?.Invalidate(); + break; + case "oracle": + oracleOptionsCache.TryRemove(Microsoft.Extensions.Options.Options.DefaultName); + sourceHttpClientOptionsCache.TryRemove(OracleOptions.HttpClientName); + break; + } + } + + public void InvalidateAll() + { + foreach (var sourceId in AdvisorySourceConfigurationDefinitions.ConfigurableSourceIds) + { + Invalidate(sourceId); + } + } +} diff --git a/src/Concelier/StellaOps.Concelier.WebService/Services/AdvisorySourceRuntimeOptionsOverlay.cs b/src/Concelier/StellaOps.Concelier.WebService/Services/AdvisorySourceRuntimeOptionsOverlay.cs new file mode 100644 index 000000000..22184f8ce --- /dev/null +++ b/src/Concelier/StellaOps.Concelier.WebService/Services/AdvisorySourceRuntimeOptionsOverlay.cs @@ -0,0 +1,170 @@ +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.Ghsa.Configuration; +using StellaOps.Concelier.Connector.Vndr.Adobe.Configuration; +using StellaOps.Concelier.Connector.Vndr.Cisco.Configuration; +using StellaOps.Concelier.Connector.Vndr.Chromium.Configuration; +using StellaOps.Concelier.Connector.Vndr.Msrc.Configuration; +using StellaOps.Concelier.Connector.Vndr.Oracle.Configuration; + +namespace StellaOps.Concelier.WebService.Services; + +public sealed class AdvisorySourceRuntimeOptionsOverlay : + IPostConfigureOptions, + IPostConfigureOptions, + IPostConfigureOptions, + IPostConfigureOptions, + IPostConfigureOptions, + IPostConfigureOptions +{ + private readonly AdvisorySourceRuntimeSettingsCache runtimeSettingsCache; + + public AdvisorySourceRuntimeOptionsOverlay(AdvisorySourceRuntimeSettingsCache runtimeSettingsCache) + { + this.runtimeSettingsCache = runtimeSettingsCache ?? throw new ArgumentNullException(nameof(runtimeSettingsCache)); + } + + public void PostConfigure(string? name, GhsaOptions options) + { + var settings = runtimeSettingsCache.GetSourceSettings("ghsa"); + if (TryGetValue(settings, "apiToken", "token", out var apiToken)) + { + options.ApiToken = apiToken; + } + } + + public void PostConfigure(string? name, AdobeOptions options) + { + var settings = runtimeSettingsCache.GetSourceSettings("adobe"); + + if (TryGetValue(settings, "indexUri", out var indexUri) && + Uri.TryCreate(indexUri, UriKind.Absolute, out var parsedIndexUri)) + { + options.IndexUri = parsedIndexUri; + } + + if (TryGetRawValue(settings, "additionalIndexUris", out var additionalIndexUris)) + { + options.AdditionalIndexUris.Clear(); + options.AdditionalIndexUris.AddRange(ParseUriList(additionalIndexUris)); + } + } + + public void PostConfigure(string? name, CiscoOptions options) + { + var settings = runtimeSettingsCache.GetSourceSettings("cisco"); + if (TryGetValue(settings, "clientId", out var clientId)) + { + options.ClientId = clientId; + } + + if (TryGetValue(settings, "clientSecret", out var clientSecret)) + { + options.ClientSecret = clientSecret; + } + } + + public void PostConfigure(string? name, ChromiumOptions options) + { + var settings = runtimeSettingsCache.GetSourceSettings("chromium"); + if (TryGetValue(settings, "feedUri", out var feedUri) && + Uri.TryCreate(feedUri, UriKind.Absolute, out var parsedFeedUri)) + { + options.FeedUri = parsedFeedUri; + } + } + + public void PostConfigure(string? name, MsrcOptions options) + { + var settings = runtimeSettingsCache.GetSourceSettings("microsoft"); + if (TryGetValue(settings, "tenantId", out var tenantId)) + { + options.TenantId = tenantId; + } + + if (TryGetValue(settings, "clientId", out var clientId)) + { + options.ClientId = clientId; + } + + if (TryGetValue(settings, "clientSecret", out var clientSecret)) + { + options.ClientSecret = clientSecret; + } + } + + public void PostConfigure(string? name, OracleOptions options) + { + var settings = runtimeSettingsCache.GetSourceSettings("oracle"); + + if (TryGetRawValue(settings, "advisoryUris", out var advisoryUris)) + { + options.AdvisoryUris = ParseUriList(advisoryUris); + } + + if (TryGetRawValue(settings, "calendarUris", out var calendarUris)) + { + options.CalendarUris = ParseUriList(calendarUris); + } + } + + private static bool TryGetValue(IReadOnlyDictionary settings, params string[] keys) + { + foreach (var key in keys) + { + if (settings.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value)) + { + return true; + } + } + + return false; + } + + private static bool TryGetValue(IReadOnlyDictionary settings, string key, out string value) + => TryGetValue(settings, [key], out value); + + private static bool TryGetValue(IReadOnlyDictionary settings, string firstKey, string secondKey, out string value) + => TryGetValue(settings, [firstKey, secondKey], out value); + + private static bool TryGetValue(IReadOnlyDictionary settings, IEnumerable keys, out string value) + { + foreach (var key in keys) + { + if (settings.TryGetValue(key, out var current) && !string.IsNullOrWhiteSpace(current)) + { + value = current.Trim(); + return true; + } + } + + value = string.Empty; + return false; + } + + private static bool TryGetRawValue(IReadOnlyDictionary settings, string key, out string value) + { + if (settings.TryGetValue(key, out var rawValue)) + { + value = rawValue ?? string.Empty; + return true; + } + + value = string.Empty; + return false; + } + + private static List ParseUriList(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return []; + } + + return value + .Split(["\r\n", "\n", ",", ";"], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Where(static item => Uri.TryCreate(item, UriKind.Absolute, out _)) + .Select(static item => new Uri(item, UriKind.Absolute)) + .DistinctBy(static uri => uri.AbsoluteUri, StringComparer.OrdinalIgnoreCase) + .ToList(); + } +} diff --git a/src/Concelier/StellaOps.Concelier.WebService/Services/AdvisorySourceRuntimeSettingsCache.cs b/src/Concelier/StellaOps.Concelier.WebService/Services/AdvisorySourceRuntimeSettingsCache.cs new file mode 100644 index 000000000..ee6e60278 --- /dev/null +++ b/src/Concelier/StellaOps.Concelier.WebService/Services/AdvisorySourceRuntimeSettingsCache.cs @@ -0,0 +1,91 @@ +using System.Collections.Immutable; +using System.Text.Json; +using StellaOps.Concelier.Core.Sources; +using StellaOps.Concelier.Persistence.Postgres.Models; + +namespace StellaOps.Concelier.WebService.Services; + +public sealed class AdvisorySourceRuntimeSettingsCache +{ + private readonly object syncRoot = new(); + private ImmutableDictionary> settingsBySource = + ImmutableDictionary>.Empty + .WithComparers(StringComparer.OrdinalIgnoreCase); + + public IReadOnlyDictionary GetSourceSettings(string sourceId) + { + var normalizedSourceId = SourceKeyAliases.Normalize(sourceId); + lock (syncRoot) + { + return settingsBySource.TryGetValue(normalizedSourceId, out var settings) + ? settings + : ImmutableDictionary.Empty.WithComparers(StringComparer.OrdinalIgnoreCase); + } + } + + public void Replace(IEnumerable sources) + { + ArgumentNullException.ThrowIfNull(sources); + + var next = ImmutableDictionary.CreateBuilder>(StringComparer.OrdinalIgnoreCase); + foreach (var source in sources) + { + var normalizedSourceId = SourceKeyAliases.Normalize(source.Key); + next[normalizedSourceId] = ParseSettings(source.Config); + } + + lock (syncRoot) + { + settingsBySource = next.ToImmutable(); + } + } + + public void Upsert(string sourceId, string? configJson) + { + var normalizedSourceId = SourceKeyAliases.Normalize(sourceId); + var parsed = ParseSettings(configJson); + + lock (syncRoot) + { + settingsBySource = settingsBySource.SetItem(normalizedSourceId, parsed); + } + } + + private static ImmutableDictionary ParseSettings(string? json) + { + if (string.IsNullOrWhiteSpace(json)) + { + return ImmutableDictionary.Empty.WithComparers(StringComparer.OrdinalIgnoreCase); + } + + try + { + using var document = JsonDocument.Parse(json); + if (document.RootElement.ValueKind != JsonValueKind.Object) + { + return ImmutableDictionary.Empty.WithComparers(StringComparer.OrdinalIgnoreCase); + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.OrdinalIgnoreCase); + foreach (var property in document.RootElement.EnumerateObject()) + { + builder[property.Name] = property.Value.ValueKind switch + { + JsonValueKind.String => property.Value.GetString() ?? string.Empty, + JsonValueKind.True => bool.TrueString, + JsonValueKind.False => bool.FalseString, + JsonValueKind.Number => property.Value.GetRawText(), + JsonValueKind.Array => property.Value.GetRawText(), + JsonValueKind.Object => property.Value.GetRawText(), + _ => string.Empty, + }; + } + + return builder.ToImmutable(); + } + catch + { + return ImmutableDictionary.Empty.WithComparers(StringComparer.OrdinalIgnoreCase); + } + } +} diff --git a/src/Concelier/StellaOps.Concelier.WebService/Services/AdvisorySourceRuntimeSettingsLoader.cs b/src/Concelier/StellaOps.Concelier.WebService/Services/AdvisorySourceRuntimeSettingsLoader.cs new file mode 100644 index 000000000..dbd5ed20a --- /dev/null +++ b/src/Concelier/StellaOps.Concelier.WebService/Services/AdvisorySourceRuntimeSettingsLoader.cs @@ -0,0 +1,44 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using StellaOps.Concelier.Persistence.Postgres.Repositories; + +namespace StellaOps.Concelier.WebService.Services; + +public sealed class AdvisorySourceRuntimeSettingsLoader : IHostedService +{ + private readonly ISourceRepository sourceRepository; + private readonly AdvisorySourceRuntimeSettingsCache runtimeSettingsCache; + private readonly AdvisorySourceRuntimeOptionsInvalidator runtimeOptionsInvalidator; + private readonly ILogger logger; + + public AdvisorySourceRuntimeSettingsLoader( + ISourceRepository sourceRepository, + AdvisorySourceRuntimeSettingsCache runtimeSettingsCache, + AdvisorySourceRuntimeOptionsInvalidator runtimeOptionsInvalidator, + ILogger logger) + { + this.sourceRepository = sourceRepository ?? throw new ArgumentNullException(nameof(sourceRepository)); + this.runtimeSettingsCache = runtimeSettingsCache ?? throw new ArgumentNullException(nameof(runtimeSettingsCache)); + this.runtimeOptionsInvalidator = runtimeOptionsInvalidator ?? throw new ArgumentNullException(nameof(runtimeOptionsInvalidator)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + try + { + var sources = await sourceRepository.ListAsync(enabled: null, cancellationToken).ConfigureAwait(false); + runtimeSettingsCache.Replace(sources); + foreach (var source in sources) + { + runtimeOptionsInvalidator.Invalidate(source.Key); + } + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed loading persisted advisory source runtime settings during startup."); + } + } + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; +} diff --git a/src/Concelier/StellaOps.Concelier.WebService/Services/ConfiguredAdvisorySourceService.cs b/src/Concelier/StellaOps.Concelier.WebService/Services/ConfiguredAdvisorySourceService.cs index 4606c995e..46630087a 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Services/ConfiguredAdvisorySourceService.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Services/ConfiguredAdvisorySourceService.cs @@ -4,8 +4,15 @@ using System.Net; using System.Net.Http; using System.Security.Authentication; using System.Text.Json; +using System.Threading; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.Ghsa.Configuration; +using StellaOps.Concelier.Connector.Vndr.Cisco.Configuration; +using StellaOps.Concelier.Connector.Vndr.Msrc.Configuration; +using StellaOps.Concelier.Connector.Vndr.Oracle.Configuration; using StellaOps.Concelier.Core.Sources; +using StellaOps.Concelier.Core.Jobs; using StellaOps.Concelier.Persistence.Postgres.Models; using StellaOps.Concelier.Persistence.Postgres.Repositories; using StellaOps.Concelier.WebService.Extensions; @@ -29,8 +36,16 @@ public sealed class ConfiguredAdvisorySourceService private readonly IHttpClientFactory httpClientFactory; private readonly IMirrorConfigStore mirrorConfigStore; private readonly IMirrorConsumerConfigStore mirrorConsumerConfigStore; + private readonly IOptionsMonitor schedulerOptions; + private readonly IOptionsMonitor ghsaOptions; + private readonly IOptionsMonitor ciscoOptions; + private readonly IOptionsMonitor msrcOptions; + private readonly IOptionsMonitor oracleOptions; + private readonly AdvisorySourceRuntimeSettingsCache runtimeSettingsCache; + private readonly AdvisorySourceRuntimeOptionsInvalidator runtimeOptionsInvalidator; private readonly TimeProvider timeProvider; private readonly ILogger logger; + private readonly SemaphoreSlim runtimeRefreshLock = new(1, 1); public ConfiguredAdvisorySourceService( ISourceRepository sourceRepository, @@ -38,6 +53,13 @@ public sealed class ConfiguredAdvisorySourceService IHttpClientFactory httpClientFactory, IMirrorConfigStore mirrorConfigStore, IMirrorConsumerConfigStore mirrorConsumerConfigStore, + IOptionsMonitor schedulerOptions, + IOptionsMonitor ghsaOptions, + IOptionsMonitor ciscoOptions, + IOptionsMonitor msrcOptions, + IOptionsMonitor oracleOptions, + AdvisorySourceRuntimeSettingsCache runtimeSettingsCache, + AdvisorySourceRuntimeOptionsInvalidator runtimeOptionsInvalidator, TimeProvider timeProvider, ILogger logger) { @@ -46,46 +68,258 @@ public sealed class ConfiguredAdvisorySourceService this.httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); this.mirrorConfigStore = mirrorConfigStore ?? throw new ArgumentNullException(nameof(mirrorConfigStore)); this.mirrorConsumerConfigStore = mirrorConsumerConfigStore ?? throw new ArgumentNullException(nameof(mirrorConsumerConfigStore)); + this.schedulerOptions = schedulerOptions ?? throw new ArgumentNullException(nameof(schedulerOptions)); + this.ghsaOptions = ghsaOptions ?? throw new ArgumentNullException(nameof(ghsaOptions)); + this.ciscoOptions = ciscoOptions ?? throw new ArgumentNullException(nameof(ciscoOptions)); + this.msrcOptions = msrcOptions ?? throw new ArgumentNullException(nameof(msrcOptions)); + this.oracleOptions = oracleOptions ?? throw new ArgumentNullException(nameof(oracleOptions)); + this.runtimeSettingsCache = runtimeSettingsCache ?? throw new ArgumentNullException(nameof(runtimeSettingsCache)); + this.runtimeOptionsInvalidator = runtimeOptionsInvalidator ?? throw new ArgumentNullException(nameof(runtimeOptionsInvalidator)); this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public async Task> GetStatusAsync(CancellationToken cancellationToken = default) { + await RefreshRuntimeSettingsAsync(cancellationToken).ConfigureAwait(false); var persisted = await sourceRepository.ListAsync(enabled: null, cancellationToken).ConfigureAwait(false); - var persistedByKey = persisted.ToDictionary(source => source.Key, StringComparer.OrdinalIgnoreCase); + var enabledByKey = persisted + .GroupBy(source => NormalizeSourceId(source.Key), StringComparer.OrdinalIgnoreCase) + .ToDictionary( + static group => group.Key, + static group => group.Any(source => source.Enabled), + StringComparer.OrdinalIgnoreCase); return sourceRegistry.GetAllSources() - .Select(source => new ConfiguredAdvisorySourceStatus( - source.Id, - persistedByKey.TryGetValue(source.Id, out var entity) && entity.Enabled, - sourceRegistry.GetLastCheckResult(source.Id))) + .Select(source => + { + var readinessFailure = GetConfigurationFailureCore(source.Id); + return new ConfiguredAdvisorySourceStatus( + source.Id, + enabledByKey.TryGetValue(source.Id, out var enabled) && + enabled && + IsSourceRunnable(source.Id) && + readinessFailure is null, + readinessFailure ?? sourceRegistry.GetLastCheckResult(source.Id), + IsSourceRunnable(source.Id), + BuildFetchJobKind(source.Id)); + }) + .ToImmutableArray(); + } + + public async Task> GetPersistedEnabledSourceIdsAsync(CancellationToken cancellationToken = default) + { + var persisted = await sourceRepository.ListAsync(enabled: true, cancellationToken).ConfigureAwait(false); + return persisted + .Select(source => NormalizeSourceId(source.Key)) + .Where(static key => !string.IsNullOrWhiteSpace(key)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static key => key, StringComparer.OrdinalIgnoreCase) .ToImmutableArray(); } public async Task> GetEnabledSourceIdsAsync(CancellationToken cancellationToken = default) { - var persisted = await sourceRepository.ListAsync(enabled: true, cancellationToken).ConfigureAwait(false); + await RefreshRuntimeSettingsAsync(cancellationToken).ConfigureAwait(false); + var persisted = await GetPersistedEnabledSourceIdsAsync(cancellationToken).ConfigureAwait(false); return persisted - .Select(source => source.Key) - .Distinct(StringComparer.OrdinalIgnoreCase) - .OrderBy(key => key, StringComparer.OrdinalIgnoreCase) + .Where(sourceId => IsSourceReadyForSyncCore(sourceId)) .ToImmutableArray(); } + public bool IsSourceRunnable(string sourceId) + { + if (string.IsNullOrWhiteSpace(sourceId)) + { + return false; + } + + return schedulerOptions.CurrentValue.Definitions.ContainsKey(BuildFetchJobKind(sourceId)); + } + + public string GetFetchJobKind(string sourceId) => BuildFetchJobKind(sourceId); + + public string NormalizeSourceId(string sourceId) => SourceKeyAliases.Normalize(sourceId); + + public bool SupportsConfiguration(string sourceId) + => AdvisorySourceConfigurationDefinitions.Get(sourceId) is not null; + + public async Task GetSourceConfigurationAsync( + string sourceId, + CancellationToken cancellationToken = default) + { + var normalizedSourceId = NormalizeSourceId(sourceId); + var sourceDefinition = sourceRegistry.GetSource(normalizedSourceId); + var schema = AdvisorySourceConfigurationDefinitions.Get(normalizedSourceId); + if (sourceDefinition is null || schema is null) + { + return null; + } + + await RefreshRuntimeSettingsAsync(cancellationToken).ConfigureAwait(false); + + var settings = runtimeSettingsCache.GetSourceSettings(normalizedSourceId); + var fields = schema.Fields + .Select(field => BuildFieldState(field, settings)) + .ToImmutableArray(); + + return new AdvisorySourceConfigurationSnapshot( + normalizedSourceId, + sourceDefinition.DisplayName, + fields); + } + + public async Task UpdateSourceConfigurationAsync( + string sourceId, + IReadOnlyDictionary? values, + IReadOnlyCollection? clearKeys, + CancellationToken cancellationToken = default) + { + var normalizedSourceId = NormalizeSourceId(sourceId); + var definition = sourceRegistry.GetSource(normalizedSourceId); + var schema = AdvisorySourceConfigurationDefinitions.Get(normalizedSourceId); + if (definition is null || schema is null) + { + return null; + } + + await RefreshRuntimeSettingsAsync(cancellationToken).ConfigureAwait(false); + + var existing = await FindPersistedSourceAsync(normalizedSourceId, cancellationToken).ConfigureAwait(false); + await UpsertSourceAsync( + definition, + enabled: existing?.Enabled ?? false, + existing, + values, + ManualMode, + existing?.Url ?? definition.BaseEndpoint, + configuredBy: "source-configuration", + cancellationToken, + clearKeys, + schema).ConfigureAwait(false); + + return await GetSourceConfigurationAsync(normalizedSourceId, cancellationToken).ConfigureAwait(false); + } + + public async Task GetConfigurationFailureAsync( + string sourceId, + CancellationToken cancellationToken = default) + { + await RefreshRuntimeSettingsAsync(cancellationToken).ConfigureAwait(false); + return GetConfigurationFailureCore(sourceId); + } + + private bool IsSourceReadyForSyncCore(string sourceId) + => IsSourceRunnable(sourceId) && GetConfigurationFailureCore(sourceId) is null; + + private SourceConnectivityResult? GetConfigurationFailureCore(string sourceId) + { + var normalizedSourceId = NormalizeSourceId(sourceId); + if (string.IsNullOrWhiteSpace(normalizedSourceId)) + { + return null; + } + + var definition = sourceRegistry.GetSource(normalizedSourceId); + if (definition is null) + { + return null; + } + + return normalizedSourceId switch + { + "ghsa" when string.IsNullOrWhiteSpace(GetOptionValue(ghsaOptions)?.ApiToken) => + CreateConfigRequiredResult( + normalizedSourceId, + "GitHub Security Advisories requires an API token before sync can run.", + ImmutableArray.Create( + "The GHSA connector is registered on this host, but no GitHub token is configured."), + ImmutableArray.Create( + new RemediationStep + { + Order = 1, + Description = "Configure a GitHub token for GHSA (for example `GITHUB_PAT` or the bound `GhsaOptions.ApiToken` setting).", + CommandType = CommandType.EnvVar + }), + definition.DocumentationUrl), + "cisco" when !HasCiscoCredentials() => + CreateConfigRequiredResult( + normalizedSourceId, + "Cisco Security requires OAuth client credentials before sync can run.", + ImmutableArray.Create( + "The Cisco connector is registered on this host, but `ClientId` and `ClientSecret` are not configured."), + ImmutableArray.Create( + new RemediationStep + { + Order = 1, + Description = "Set `CONCELIER__SOURCES__VNDR__CISCO__CLIENTID` and `CONCELIER__SOURCES__VNDR__CISCO__CLIENTSECRET` for the Cisco PSIRT connector.", + CommandType = CommandType.EnvVar + }), + definition.DocumentationUrl), + "microsoft" when !HasMsrcCredentials() => + CreateConfigRequiredResult( + normalizedSourceId, + "Microsoft Security (MSRC) requires Azure AD tenant, client ID, and client secret before sync can run.", + ImmutableArray.Create( + "The MSRC connector is registered on this host, but the required Azure AD client-credential settings are missing or invalid."), + ImmutableArray.Create( + new RemediationStep + { + Order = 1, + Description = "Set `CONCELIER__SOURCES__MICROSOFT__TENANTID`, `CONCELIER__SOURCES__MICROSOFT__CLIENTID`, and `CONCELIER__SOURCES__MICROSOFT__CLIENTSECRET` for the MSRC connector (legacy `CONCELIER__SOURCES__VNDR__MSRC__*` is still accepted).", + CommandType = CommandType.EnvVar + }), + definition.DocumentationUrl), + "oracle" when !HasOracleUris() => + CreateConfigRequiredResult( + normalizedSourceId, + "Oracle Security requires at least one advisory or calendar URI before sync can run.", + ImmutableArray.Create( + "The Oracle connector is registered on this host, but no `AdvisoryUris` or `CalendarUris` are configured."), + ImmutableArray.Create( + new RemediationStep + { + Order = 1, + Description = "Configure `CalendarUris` and/or `AdvisoryUris` for the Oracle connector. The documented default is the Oracle security alerts calendar page.", + CommandType = CommandType.EnvVar, + DocumentationUrl = definition.DocumentationUrl + }), + definition.DocumentationUrl), + _ => null + }; + } + public async Task EnableSourceAsync( string sourceId, IReadOnlyDictionary? configValues = null, CancellationToken cancellationToken = default) { - var definition = sourceRegistry.GetSource(sourceId); + await RefreshRuntimeSettingsAsync(cancellationToken).ConfigureAwait(false); + var normalizedSourceId = NormalizeSourceId(sourceId); + var definition = sourceRegistry.GetSource(normalizedSourceId); if (definition is null) { return false; } - var existing = await sourceRepository.GetByKeyAsync(sourceId, cancellationToken).ConfigureAwait(false); - var mirrorUrl = string.Equals(sourceId, MirrorSourceId, StringComparison.OrdinalIgnoreCase) + if (!IsSourceRunnable(normalizedSourceId)) + { + logger.LogWarning( + "Rejected enable request for advisory source {SourceId} because no runnable fetch job is registered.", + normalizedSourceId); + return false; + } + + if (GetConfigurationFailureCore(normalizedSourceId) is not null) + { + logger.LogWarning( + "Rejected enable request for advisory source {SourceId} because required connector configuration is missing.", + normalizedSourceId); + return false; + } + + var existing = await FindPersistedSourceAsync(normalizedSourceId, cancellationToken).ConfigureAwait(false); + var mirrorUrl = string.Equals(normalizedSourceId, MirrorSourceId, StringComparison.OrdinalIgnoreCase) ? NormalizeMirrorUrl(GetConfigValue(configValues, "sources.mirror.url") ?? existing?.Url ?? definition.BaseEndpoint) : null; await UpsertSourceAsync( @@ -93,45 +327,46 @@ public sealed class ConfiguredAdvisorySourceService enabled: true, existing, configValues, - string.Equals(sourceId, MirrorSourceId, StringComparison.OrdinalIgnoreCase) ? MirrorMode : ManualMode, + string.Equals(normalizedSourceId, MirrorSourceId, StringComparison.OrdinalIgnoreCase) ? MirrorMode : ManualMode, mirrorUrl, configuredBy: "source-management", cancellationToken).ConfigureAwait(false); - if (string.Equals(sourceId, MirrorSourceId, StringComparison.OrdinalIgnoreCase)) + if (string.Equals(normalizedSourceId, MirrorSourceId, StringComparison.OrdinalIgnoreCase)) { await ApplyMirrorConsumerConfigurationAsync(mirrorUrl!, cancellationToken).ConfigureAwait(false); } - await sourceRegistry.EnableSourceAsync(sourceId, cancellationToken).ConfigureAwait(false); + await sourceRegistry.EnableSourceAsync(normalizedSourceId, cancellationToken).ConfigureAwait(false); return true; } public async Task DisableSourceAsync(string sourceId, CancellationToken cancellationToken = default) { - var definition = sourceRegistry.GetSource(sourceId); + var normalizedSourceId = NormalizeSourceId(sourceId); + var definition = sourceRegistry.GetSource(normalizedSourceId); if (definition is null) { return false; } - var existing = await sourceRepository.GetByKeyAsync(sourceId, cancellationToken).ConfigureAwait(false); + var existing = await FindPersistedSourceAsync(normalizedSourceId, cancellationToken).ConfigureAwait(false); await UpsertSourceAsync( definition, enabled: false, existing, configValues: null, - mode: string.Equals(sourceId, MirrorSourceId, StringComparison.OrdinalIgnoreCase) ? MirrorMode : ManualMode, + mode: string.Equals(normalizedSourceId, MirrorSourceId, StringComparison.OrdinalIgnoreCase) ? MirrorMode : ManualMode, mirrorUrl: existing?.Url, configuredBy: "source-management", cancellationToken).ConfigureAwait(false); - if (string.Equals(sourceId, MirrorSourceId, StringComparison.OrdinalIgnoreCase)) + if (string.Equals(normalizedSourceId, MirrorSourceId, StringComparison.OrdinalIgnoreCase)) { await SetMirrorModeAsync(enabled: false, consumerBaseAddress: null, cancellationToken).ConfigureAwait(false); } - await sourceRegistry.DisableSourceAsync(sourceId, cancellationToken).ConfigureAwait(false); + await sourceRegistry.DisableSourceAsync(normalizedSourceId, cancellationToken).ConfigureAwait(false); return true; } @@ -139,36 +374,52 @@ public sealed class ConfiguredAdvisorySourceService string sourceId, CancellationToken cancellationToken = default) { - var definition = sourceRegistry.GetSource(sourceId); + await RefreshRuntimeSettingsAsync(cancellationToken).ConfigureAwait(false); + var normalizedSourceId = NormalizeSourceId(sourceId); + var definition = sourceRegistry.GetSource(normalizedSourceId); if (definition is null) { - return SourceConnectivityResult.NotFound(sourceId); + return SourceConnectivityResult.NotFound(string.IsNullOrWhiteSpace(normalizedSourceId) ? sourceId : normalizedSourceId); } - if (!string.Equals(sourceId, MirrorSourceId, StringComparison.OrdinalIgnoreCase)) + if (!IsSourceRunnable(normalizedSourceId)) { - return await sourceRegistry.CheckConnectivityAsync(sourceId, cancellationToken).ConfigureAwait(false); + return CreateUnsupportedResult(normalizedSourceId, definition.DocumentationUrl); } - var existing = await sourceRepository.GetByKeyAsync(sourceId, cancellationToken).ConfigureAwait(false); + var configurationFailure = GetConfigurationFailureCore(normalizedSourceId); + if (configurationFailure is not null) + { + return configurationFailure; + } + + if (!string.Equals(normalizedSourceId, MirrorSourceId, StringComparison.OrdinalIgnoreCase)) + { + return await sourceRegistry.CheckConnectivityAsync(normalizedSourceId, cancellationToken).ConfigureAwait(false); + } + + var existing = await FindPersistedSourceAsync(normalizedSourceId, cancellationToken).ConfigureAwait(false); var mirrorUrl = NormalizeMirrorUrl(existing?.Url ?? definition.BaseEndpoint); return await ProbeMirrorConnectivityAsync(mirrorUrl, cancellationToken).ConfigureAwait(false); } public async Task CheckAllAndPersistAsync(CancellationToken cancellationToken = default) { + await RefreshRuntimeSettingsAsync(cancellationToken).ConfigureAwait(false); var checkedAt = timeProvider.GetUtcNow(); var started = Stopwatch.GetTimestamp(); var results = ImmutableArray.CreateBuilder(); foreach (var definition in sourceRegistry.GetAllSources().OrderBy(source => source.Id, StringComparer.OrdinalIgnoreCase)) { - var result = await CheckSourceConnectivityAsync(definition.Id, cancellationToken).ConfigureAwait(false); + var result = IsSourceRunnable(definition.Id) + ? await CheckSourceConnectivityAsync(definition.Id, cancellationToken).ConfigureAwait(false) + : CreateUnsupportedResult(definition.Id, definition.DocumentationUrl); results.Add(result); - var existing = await sourceRepository.GetByKeyAsync(definition.Id, cancellationToken).ConfigureAwait(false); + var existing = await FindPersistedSourceAsync(definition.Id, cancellationToken).ConfigureAwait(false); await UpsertSourceAsync( definition, - enabled: result.IsHealthy, + enabled: result.IsHealthy && IsSourceReadyForSyncCore(definition.Id), existing, configValues: null, mode: string.Equals(definition.Id, MirrorSourceId, StringComparison.OrdinalIgnoreCase) ? MirrorMode : ManualMode, @@ -230,7 +481,7 @@ public sealed class ConfiguredAdvisorySourceService { var definition = sourceRegistry.GetSource(sourceId) ?? throw new InvalidOperationException($"Source '{sourceId}' is not registered."); - var existing = persisted.FirstOrDefault(source => string.Equals(source.Key, sourceId, StringComparison.OrdinalIgnoreCase)); + var existing = persisted.FirstOrDefault(source => SourceKeyAliases.Matches(source.Key, sourceId)); await UpsertSourceAsync( definition, @@ -252,7 +503,7 @@ public sealed class ConfiguredAdvisorySourceService foreach (var existing in persisted) { - if (selected.Contains(existing.Key)) + if (selected.Contains(NormalizeSourceId(existing.Key))) { continue; } @@ -334,6 +585,15 @@ public sealed class ConfiguredAdvisorySourceService throw new InvalidOperationException("StellaOps Mirror is configured through mirror mode. Switch modes instead of selecting it manually."); } + var unsupported = selected + .Where(sourceId => !IsSourceRunnable(sourceId)) + .OrderBy(sourceId => sourceId, StringComparer.OrdinalIgnoreCase) + .ToArray(); + if (unsupported.Length > 0) + { + throw new InvalidOperationException($"Selected advisory/VEX sources do not have a runnable fetch pipeline in this host: {string.Join(", ", unsupported)}."); + } + return new SetupAdvisorySourceInstruction( ManualMode, selected, @@ -378,7 +638,7 @@ public sealed class ConfiguredAdvisorySourceService Message: $"Connectivity failed for {failures.Count} selected source(s). {string.Join(" ", failures)}"); } - private async Task UpsertSourceAsync( + private async Task UpsertSourceAsync( SourceDefinition definition, bool enabled, SourceEntity? existing, @@ -386,10 +646,12 @@ public sealed class ConfiguredAdvisorySourceService string mode, string? mirrorUrl, string configuredBy, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + IReadOnlyCollection? clearKeys = null, + AdvisorySourceConfigurationSchema? schema = null) { var now = timeProvider.GetUtcNow(); - var config = BuildSourceConfig(definition, configValues, mode, mirrorUrl, existing?.Config); + var config = BuildSourceConfig(definition, configValues, mode, mirrorUrl, existing?.Config, clearKeys, schema); var metadata = BuildSourceMetadata(existing?.Metadata, configuredBy, mode); var entity = new SourceEntity @@ -410,6 +672,9 @@ public sealed class ConfiguredAdvisorySourceService }; await sourceRepository.UpsertAsync(entity, cancellationToken).ConfigureAwait(false); + runtimeSettingsCache.Upsert(entity.Key, entity.Config); + runtimeOptionsInvalidator.Invalidate(entity.Key); + return entity; } private static string BuildSourceConfig( @@ -417,17 +682,24 @@ public sealed class ConfiguredAdvisorySourceService IReadOnlyDictionary? configValues, string mode, string? mirrorUrl, - string? existingJson) + string? existingJson, + IReadOnlyCollection? clearKeys, + AdvisorySourceConfigurationSchema? schema) { var builder = ParseJsonObject(existingJson); + var clearLookup = BuildClearLookup(clearKeys); if (string.Equals(definition.Id, MirrorSourceId, StringComparison.OrdinalIgnoreCase)) { builder["mode"] = mode; builder["baseUrl"] = mirrorUrl ?? SourceDefinitions.StellaMirror.BaseEndpoint; - var apiKey = GetConfigValue(configValues, "sources.mirror.apiKey"); - if (!string.IsNullOrWhiteSpace(apiKey)) + if (ShouldClearField(clearLookup, definition.Id, "apiKey")) + { + builder.Remove("apiKey"); + } + else if (TryGetSubmittedValue(configValues, definition.Id, "apiKey", aliases: null, out _, out var apiKey) && + !string.IsNullOrWhiteSpace(apiKey)) { builder["apiKey"] = apiKey; } @@ -435,6 +707,49 @@ public sealed class ConfiguredAdvisorySourceService return JsonSerializer.Serialize(builder); } + var handledFields = new HashSet(StringComparer.OrdinalIgnoreCase); + if (schema is not null) + { + foreach (var field in schema.Fields) + { + handledFields.Add(field.Key); + foreach (var alias in field.Aliases) + { + handledFields.Add(alias); + } + + if (ShouldClearField(clearLookup, definition.Id, field.Key, field.Aliases)) + { + builder.Remove(field.Key); + continue; + } + + if (!TryGetSubmittedValue(configValues, definition.Id, field.Key, field.Aliases, out var submitted, out var value)) + { + continue; + } + + if (field.Sensitive) + { + if (!string.IsNullOrWhiteSpace(value)) + { + builder[field.Key] = value; + } + + continue; + } + + if (string.IsNullOrWhiteSpace(value)) + { + builder.Remove(field.Key); + } + else + { + builder[field.Key] = NormalizeFieldValue(field.Key, value); + } + } + } + var prefix = $"sources.{definition.Id}."; if (configValues is not null) { @@ -446,12 +761,24 @@ public sealed class ConfiguredAdvisorySourceService } var field = pair.Key[prefix.Length..]; + if (handledFields.Contains(field)) + { + continue; + } + if (string.Equals(field, "enabled", StringComparison.OrdinalIgnoreCase)) { continue; } - builder[field] = pair.Value; + if (string.IsNullOrWhiteSpace(pair.Value)) + { + builder.Remove(field); + } + else + { + builder[field] = pair.Value.Trim(); + } } } @@ -517,7 +844,7 @@ public sealed class ConfiguredAdvisorySourceService { foreach (var sourceId in enabledList.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) { - selected.Add(sourceId); + selected.Add(SourceKeyAliases.Normalize(sourceId)); } } @@ -539,11 +866,11 @@ public sealed class ConfiguredAdvisorySourceService { if (enabled) { - selected.Add(sourceId); + selected.Add(SourceKeyAliases.Normalize(sourceId)); } else { - selected.Remove(sourceId); + selected.Remove(SourceKeyAliases.Normalize(sourceId)); } } } @@ -558,6 +885,26 @@ public sealed class ConfiguredAdvisorySourceService ? ManualMode : MirrorMode; + private static string BuildFetchJobKind(string sourceId) => $"source:{SourceKeyAliases.Normalize(sourceId)}:fetch"; + + private static AdvisorySourceConfigurationFieldState BuildFieldState( + AdvisorySourceConfigurationFieldDefinition field, + IReadOnlyDictionary settings) + { + var hasValue = TryGetStoredValue(settings, field.Key, field.Aliases, out var value); + return new AdvisorySourceConfigurationFieldState( + field.Key, + field.Label, + field.InputType, + field.Sensitive, + field.Required, + field.Sensitive ? null : value, + hasValue, + field.Sensitive && hasValue, + field.HelpText, + field.Placeholder); + } + private static string? GetConfigValue(IReadOnlyDictionary? configValues, string key) { if (configValues is null) @@ -570,6 +917,242 @@ public sealed class ConfiguredAdvisorySourceService : null; } + private static bool TryGetStoredValue( + IReadOnlyDictionary settings, + string key, + IEnumerable aliases, + out string? value) + { + if (settings.TryGetValue(key, out var current)) + { + value = current; + return !string.IsNullOrWhiteSpace(current); + } + + foreach (var alias in aliases) + { + if (settings.TryGetValue(alias, out current)) + { + value = current; + return !string.IsNullOrWhiteSpace(current); + } + } + + value = null; + return false; + } + + private static Dictionary> BuildClearLookup(IReadOnlyCollection? clearKeys) + { + var lookup = new Dictionary>(StringComparer.OrdinalIgnoreCase); + if (clearKeys is null) + { + return lookup; + } + + foreach (var key in clearKeys) + { + if (string.IsNullOrWhiteSpace(key)) + { + continue; + } + + var trimmed = key.Trim(); + if (!lookup.TryGetValue(string.Empty, out var shared)) + { + shared = new HashSet(StringComparer.OrdinalIgnoreCase); + lookup[string.Empty] = shared; + } + + shared.Add(trimmed); + + if (trimmed.StartsWith("sources.", StringComparison.OrdinalIgnoreCase)) + { + var remainder = trimmed["sources.".Length..]; + var separatorIndex = remainder.IndexOf('.'); + if (separatorIndex > 0 && separatorIndex < remainder.Length - 1) + { + var sourceId = SourceKeyAliases.Normalize(remainder[..separatorIndex]); + var fieldKey = remainder[(separatorIndex + 1)..]; + if (!lookup.TryGetValue(sourceId, out var scoped)) + { + scoped = new HashSet(StringComparer.OrdinalIgnoreCase); + lookup[sourceId] = scoped; + } + + scoped.Add(fieldKey); + } + } + } + + return lookup; + } + + private static bool ShouldClearField( + IReadOnlyDictionary> clearLookup, + string sourceId, + string fieldKey, + IEnumerable? aliases = null) + { + var normalizedSourceId = SourceKeyAliases.Normalize(sourceId); + var candidates = new HashSet(StringComparer.OrdinalIgnoreCase) + { + fieldKey, + $"sources.{normalizedSourceId}.{fieldKey}", + }; + + if (aliases is not null) + { + foreach (var alias in aliases) + { + candidates.Add(alias); + candidates.Add($"sources.{normalizedSourceId}.{alias}"); + } + } + + return clearLookup.TryGetValue(string.Empty, out var shared) && candidates.Any(shared.Contains) + || clearLookup.TryGetValue(normalizedSourceId, out var scoped) && (scoped.Contains(fieldKey) || (aliases?.Any(scoped.Contains) ?? false)); + } + + private static bool TryGetSubmittedValue( + IReadOnlyDictionary? configValues, + string sourceId, + string fieldKey, + IEnumerable? aliases, + out bool submitted, + out string? value) + { + var normalizedSourceId = SourceKeyAliases.Normalize(sourceId); + foreach (var key in EnumerateSubmittedKeys(normalizedSourceId, fieldKey, aliases)) + { + if (configValues is not null && configValues.TryGetValue(key, out var current)) + { + submitted = true; + value = current.Trim(); + return true; + } + } + + submitted = false; + value = null; + return false; + } + + private static IEnumerable EnumerateSubmittedKeys(string sourceId, string fieldKey, IEnumerable? aliases) + { + yield return $"sources.{sourceId}.{fieldKey}"; + yield return fieldKey; + + if (aliases is null) + { + yield break; + } + + foreach (var alias in aliases) + { + yield return $"sources.{sourceId}.{alias}"; + yield return alias; + } + } + + private static string NormalizeFieldValue(string fieldKey, string value) + => string.Equals(fieldKey, "advisoryUris", StringComparison.OrdinalIgnoreCase) || + string.Equals(fieldKey, "calendarUris", StringComparison.OrdinalIgnoreCase) || + string.Equals(fieldKey, "additionalIndexUris", StringComparison.OrdinalIgnoreCase) + ? NormalizeMultilineList(value) + : value.Trim(); + + private static string NormalizeMultilineList(string value) + => string.Join( + Environment.NewLine, + value + .Split(["\r\n", "\n", ",", ";"], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Distinct(StringComparer.OrdinalIgnoreCase)); + + private async Task FindPersistedSourceAsync(string sourceId, CancellationToken cancellationToken) + { + foreach (var candidate in SourceKeyAliases.GetEquivalentKeys(sourceId)) + { + var existing = await sourceRepository.GetByKeyAsync(candidate, cancellationToken).ConfigureAwait(false); + if (existing is not null) + { + return existing; + } + } + + return null; + } + + private async Task RefreshRuntimeSettingsAsync(CancellationToken cancellationToken) + { + await runtimeRefreshLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var persisted = await sourceRepository.ListAsync(enabled: null, cancellationToken).ConfigureAwait(false); + runtimeSettingsCache.Replace(persisted); + runtimeOptionsInvalidator.InvalidateAll(); + } + finally + { + runtimeRefreshLock.Release(); + } + } + + private static TOptions? GetOptionValue(IOptionsMonitor options) + where TOptions : class + { + try + { + return options.CurrentValue; + } + catch + { + return null; + } + } + + private bool HasCiscoCredentials() + { + var options = GetOptionValue(ciscoOptions); + return options is not null && + !string.IsNullOrWhiteSpace(options.ClientId) && + !string.IsNullOrWhiteSpace(options.ClientSecret); + } + + private bool HasMsrcCredentials() + { + var options = GetOptionValue(msrcOptions); + return options is not null && + Guid.TryParse(options.TenantId, out _) && + !string.IsNullOrWhiteSpace(options.ClientId) && + !string.IsNullOrWhiteSpace(options.ClientSecret); + } + + private bool HasOracleUris() + { + var options = GetOptionValue(oracleOptions); + return options is not null && (options.AdvisoryUris.Count > 0 || options.CalendarUris.Count > 0); + } + + private SourceConnectivityResult CreateConfigRequiredResult( + string sourceId, + string message, + ImmutableArray possibleReasons, + ImmutableArray remediationSteps, + string? documentationUrl) + => SourceConnectivityResult.Failed( + sourceId, + "SOURCE_CONFIG_REQUIRED", + message, + possibleReasons, + remediationSteps, + checkedAt: timeProvider.GetUtcNow()) with + { + DocumentationUrl = documentationUrl, + Diagnostics = ImmutableDictionary.Empty + .Add("fetchJobKind", BuildFetchJobKind(sourceId)) + }; + private async Task ProbeMirrorConnectivityAsync( string mirrorUrl, CancellationToken cancellationToken) @@ -784,6 +1367,33 @@ public sealed class ConfiguredAdvisorySourceService return current.Message; } + private SourceConnectivityResult CreateUnsupportedResult(string sourceId, string? documentationUrl) + => SourceConnectivityResult.Failed( + sourceId, + "SOURCE_UNSUPPORTED", + $"Source '{sourceId}' is defined in the catalog but this Concelier host does not register a runnable fetch job for it.", + ImmutableArray.Create( + "This source is catalog-only in the current deployment.", + "The host is missing a connector implementation or job registration for this source."), + ImmutableArray.Create( + new RemediationStep + { + Order = 1, + Description = "Enable or ship a connector that registers the matching source::fetch job." + }, + new RemediationStep + { + Order = 2, + Description = "Keep the source disabled until the connector runtime is available.", + DocumentationUrl = documentationUrl + }), + checkedAt: timeProvider.GetUtcNow()) with + { + DocumentationUrl = documentationUrl, + Diagnostics = ImmutableDictionary.Empty + .Add("fetchJobKind", BuildFetchJobKind(sourceId)) + }; + private sealed record SetupAdvisorySourceInstruction( string Mode, ImmutableArray EnabledSourceIds, @@ -798,7 +1408,9 @@ public sealed class ConfiguredAdvisorySourceService public sealed record ConfiguredAdvisorySourceStatus( string SourceId, bool Enabled, - SourceConnectivityResult? LastCheck); + SourceConnectivityResult? LastCheck, + bool SyncSupported, + string FetchJobKind); public sealed record SetupAdvisorySourcesBootstrapRequest( IReadOnlyDictionary? ConfigValues); diff --git a/src/Concelier/StellaOps.Concelier.WebService/Services/UnsupportedJobCoordinator.cs b/src/Concelier/StellaOps.Concelier.WebService/Services/UnsupportedJobCoordinator.cs deleted file mode 100644 index 28dbc1a44..000000000 --- a/src/Concelier/StellaOps.Concelier.WebService/Services/UnsupportedJobCoordinator.cs +++ /dev/null @@ -1,30 +0,0 @@ -using StellaOps.Concelier.Core.Jobs; - -namespace StellaOps.Concelier.WebService.Services; - -public sealed class UnsupportedJobCoordinator : IJobCoordinator -{ - public const string Detail = - "Concelier job scheduling and run history require a durable backend implementation; the live runtime no longer falls back to in-memory job state."; - - public Task TriggerAsync(string kind, IReadOnlyDictionary? parameters, string trigger, CancellationToken cancellationToken) - => throw new NotSupportedException(Detail); - - public Task> GetDefinitionsAsync(CancellationToken cancellationToken) - => throw new NotSupportedException(Detail); - - public Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken) - => throw new NotSupportedException(Detail); - - public Task> GetActiveRunsAsync(CancellationToken cancellationToken) - => throw new NotSupportedException(Detail); - - public Task GetRunAsync(Guid runId, CancellationToken cancellationToken) - => throw new NotSupportedException(Detail); - - public Task GetLastRunAsync(string kind, CancellationToken cancellationToken) - => throw new NotSupportedException(Detail); - - public Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken) - => throw new NotSupportedException(Detail); -} diff --git a/src/Concelier/StellaOps.Concelier.WebService/Services/UnsupportedOrchestratorRegistryStore.cs b/src/Concelier/StellaOps.Concelier.WebService/Services/UnsupportedOrchestratorRegistryStore.cs deleted file mode 100644 index d4b229e2b..000000000 --- a/src/Concelier/StellaOps.Concelier.WebService/Services/UnsupportedOrchestratorRegistryStore.cs +++ /dev/null @@ -1,49 +0,0 @@ -using StellaOps.Concelier.Core.Orchestration; - -namespace StellaOps.Concelier.WebService.Services; - -public sealed class UnsupportedOrchestratorRegistryStore : IOrchestratorRegistryStore -{ - public const string Detail = - "Concelier internal orchestrator registry and command state require a durable backend implementation; the live runtime no longer falls back to in-memory registry storage."; - - public Task UpsertAsync(OrchestratorRegistryRecord record, CancellationToken cancellationToken) - => throw new NotSupportedException(Detail); - - public Task GetAsync(string tenant, string connectorId, CancellationToken cancellationToken) - => throw new NotSupportedException(Detail); - - public Task> ListAsync(string tenant, CancellationToken cancellationToken) - => throw new NotSupportedException(Detail); - - public Task AppendHeartbeatAsync(OrchestratorHeartbeatRecord heartbeat, CancellationToken cancellationToken) - => throw new NotSupportedException(Detail); - - public Task GetLatestHeartbeatAsync( - string tenant, - string connectorId, - Guid runId, - CancellationToken cancellationToken) - => throw new NotSupportedException(Detail); - - public Task EnqueueCommandAsync(OrchestratorCommandRecord command, CancellationToken cancellationToken) - => throw new NotSupportedException(Detail); - - public Task> GetPendingCommandsAsync( - string tenant, - string connectorId, - Guid runId, - long? afterSequence, - CancellationToken cancellationToken) - => throw new NotSupportedException(Detail); - - public Task StoreManifestAsync(OrchestratorRunManifest manifest, CancellationToken cancellationToken) - => throw new NotSupportedException(Detail); - - public Task GetManifestAsync( - string tenant, - string connectorId, - Guid runId, - CancellationToken cancellationToken) - => throw new NotSupportedException(Detail); -} diff --git a/src/Concelier/StellaOps.Concelier.WebService/StellaOps.Concelier.WebService.csproj b/src/Concelier/StellaOps.Concelier.WebService/StellaOps.Concelier.WebService.csproj index d716a486a..207f21343 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/StellaOps.Concelier.WebService.csproj +++ b/src/Concelier/StellaOps.Concelier.WebService/StellaOps.Concelier.WebService.csproj @@ -31,6 +31,8 @@ + + @@ -54,7 +56,9 @@ + + diff --git a/src/Concelier/StellaOps.Concelier.WebService/TASKS.md b/src/Concelier/StellaOps.Concelier.WebService/TASKS.md index 22fb471ed..c21fb46b9 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/TASKS.md +++ b/src/Concelier/StellaOps.Concelier.WebService/TASKS.md @@ -1,7 +1,7 @@ # Concelier WebService Task Board This board mirrors active sprint tasks for this module. -Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229_049_BE_csproj_audit_maint_tests.md`. +Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229_049_BE_csproj_audit_maint_tests.md`, `docs/implplan/SPRINT_20260408_004_Timeline_unified_audit_sink.md`. | Task ID | Status | Notes | | --- | --- | --- | @@ -15,5 +15,9 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | REALPLAN-007-C | DONE | 2026-04-15: Removed the live `InMemoryLeaseStore` binding; WebService now relies on the Postgres-backed lease store from `AddConcelierPostgresStorage`. | | REALPLAN-007-D | DONE | 2026-04-19: Removed hidden live `InMemoryJobStore`/`InMemoryOrchestratorRegistryStore` fallbacks; `/jobs`, `/internal/orch/*`, and coordinator-backed manual sync compatibility routes now return explicit `501` until durable runtime storage exists. | | REALPLAN-007-E | DONE | 2026-04-17: Removed the live `InMemoryAffectedSymbolStore` fallback; `/v1/signals/symbols/*` now returns explicit `501` outside `Testing` until a durable affected-symbol backend exists. | -| REALPLAN-007-F | DOING | 2026-04-19: Replacing the truthful `501` affected-symbol fallback with durable PostgreSQL-backed advisory-observation and affected-symbol runtime services fed from the real raw-ingest path. | +| REALPLAN-007-F | DONE | 2026-04-19: Replaced the truthful `501` affected-symbol fallback with durable PostgreSQL-backed advisory-observation and affected-symbol runtime services fed from the real raw-ingest path. Verified with targeted xUnit helper coverage for `ConcelierInfrastructureRegistrationTests`, `PostgresAdvisoryObservationStoreTests`, `PostgresAffectedSymbolStoreTests`, and `UnsupportedRuntimeWiringTests`. | +| REALPLAN-007-G | DONE | 2026-04-19: Added durable PostgreSQL-backed job-run and internal orchestrator runtime storage so live `/jobs`, `/internal/orch/*`, and coordinator-backed manual sync compatibility routes resolve `PostgresJobStore`, `JobCoordinator`, and `PostgresOrchestratorRegistryStore` outside `Testing`. Verified with the focused `UnsupportedRuntimeWiringTests` cutover coverage after rebuilding the test host assembly. | +| NOMOCK-027 | DONE | 2026-04-20: Retired the host-owned `Testing` `AddInMemoryStorage()` bootstrap and the implicit `"inmemory"` advisory-observation event transport default from `Program.cs`; explicit web-service test factories now own the remaining compatibility wiring. | | ADV-SETUP-002 | DONE | `docs/implplan/SPRINT_20260417_001_Platform_setup_advisory_vex_onboarding.md`: made advisory source enablement/status durable and exposed setup bootstrap orchestration for source onboarding. | +| AUDIT-002-CONCELIER | DONE | 2026-04-19: Closed the remaining Concelier audit metadata gaps across topology setup, advisory-source check/sync orchestration, and internal orchestrator/event-publish routes. | +| FE-ADVISORY-003-SUPPORT | DONE | 2026-04-21: `/api/v1/advisory-sources` now ships explicit review counters for source documents, canonical advisories, CVEs, and VEX documents so Integrations and Security can render the same live operator totals. | diff --git a/src/Concelier/StellaOps.Excititor.WebService/Program.cs b/src/Concelier/StellaOps.Excititor.WebService/Program.cs index 0eeae24b3..3c4a133b4 100644 --- a/src/Concelier/StellaOps.Excititor.WebService/Program.cs +++ b/src/Concelier/StellaOps.Excititor.WebService/Program.cs @@ -13,7 +13,13 @@ using StellaOps.Excititor.Attestation; using StellaOps.Excititor.Attestation.Extensions; using StellaOps.Excititor.Attestation.Transparency; using StellaOps.Excititor.Attestation.Verification; +using StellaOps.Excititor.Connectors.Cisco.CSAF.DependencyInjection; +using StellaOps.Excititor.Connectors.MSRC.CSAF.DependencyInjection; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.DependencyInjection; +using StellaOps.Excititor.Connectors.Oracle.CSAF.DependencyInjection; using StellaOps.Excititor.Connectors.RedHat.CSAF.DependencyInjection; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.DependencyInjection; +using StellaOps.Excititor.Connectors.Ubuntu.CSAF.DependencyInjection; using StellaOps.Excititor.Core; using StellaOps.Excititor.Core.Aoc; using StellaOps.Excititor.Core.Evidence; @@ -60,6 +66,7 @@ services.AddOptions() .Bind(configuration.GetSection("Excititor:Graph")); services.AddExcititorPersistence(configuration); +services.AddVexNormalization(); services.AddCsafNormalizer(); services.AddCycloneDxNormalizer(); services.AddOpenVexNormalizer(); @@ -116,6 +123,16 @@ services.AddSingleton(); // EXCITITOR-VULN-29-004: Normalization observability for Vuln Explorer + Advisory AI dashboards services.AddSingleton(); services.AddRedHatCsafConnector(); +services.AddUbuntuCsafConnector(); +services.AddOracleCsafConnector(); +services.AddCiscoCsafConnector(); +services.AddRancherHubConnector(); +services.AddOciOpenVexAttestationConnector(); +var msrcConnectorSection = configuration.GetSection("Excititor").GetSection("Connectors").GetSection("Msrc"); +if (msrcConnectorSection.Exists()) +{ + services.AddMsrcCsafConnector(options => msrcConnectorSection.Bind(options)); +} services.Configure(configuration.GetSection(MirrorDistributionOptions.SectionName)); services.AddSingleton(); services.TryAddSingleton(TimeProvider.System); @@ -176,7 +193,7 @@ services.AddSingleton(TimeProvider.System); services.AddMemoryCache(); // Register authentication services so app.UseAuthentication() can resolve IAuthenticationSchemeProvider. -services.AddStellaOpsResourceServerAuthentication(builder.Configuration); +ConfigureExcititorResourceServerAuthentication(services, configuration); // Unified audit emission (posts audit events to Timeline service). See SPRINT_20260408_004 AUDIT-002. services.AddAuditEmission(builder.Configuration); @@ -2374,6 +2391,87 @@ app.TryRefreshStellaRouterEndpoints(routerEnabled); await app.LoadTranslationsAsync(); app.Run(); +static void ConfigureExcititorResourceServerAuthentication(IServiceCollection services, IConfiguration configuration) +{ + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddStellaOpsResourceServerAuthentication( + configuration, + configure: resourceOptions => + { + var resourceSection = configuration.GetSection("Authority").GetSection("ResourceServer"); + var authority = FirstNonEmpty( + resourceOptions.Authority, + resourceSection["Authority"], + configuration["Excititor:Authority:BaseUrls:default"], + configuration["AUTHORITY_ISSUER"]); + + if (!string.IsNullOrWhiteSpace(authority) && string.IsNullOrWhiteSpace(resourceOptions.Authority)) + { + resourceOptions.Authority = authority; + } + + var metadataAddress = FirstNonEmpty( + resourceOptions.MetadataAddress, + resourceSection["MetadataAddress"], + BuildMetadataAddress(resourceOptions.Authority)); + if (!string.IsNullOrWhiteSpace(metadataAddress) && string.IsNullOrWhiteSpace(resourceOptions.MetadataAddress)) + { + resourceOptions.MetadataAddress = metadataAddress; + } + + AddDistinct(resourceOptions.Audiences, resourceSection.GetSection("Audiences").Get()); + AddDistinct(resourceOptions.RequiredScopes, resourceSection.GetSection("RequiredScopes").Get()); + AddDistinct(resourceOptions.RequiredTenants, resourceSection.GetSection("RequiredTenants").Get()); + AddDistinct(resourceOptions.BypassNetworks, resourceSection.GetSection("BypassNetworks").Get()); + }); +} + +static string? FirstNonEmpty(params string?[] values) +{ + foreach (var value in values) + { + if (!string.IsNullOrWhiteSpace(value)) + { + return value.Trim(); + } + } + + return null; +} + +static string? BuildMetadataAddress(string? authority) +{ + if (string.IsNullOrWhiteSpace(authority) || !Uri.TryCreate(authority, UriKind.Absolute, out var authorityUri)) + { + return null; + } + + return new Uri(authorityUri, ".well-known/openid-configuration").ToString(); +} + +static void AddDistinct(IList target, IEnumerable? values) +{ + if (values is null) + { + return; + } + + foreach (var value in values) + { + if (string.IsNullOrWhiteSpace(value)) + { + continue; + } + + if (!target.Contains(value, StringComparer.OrdinalIgnoreCase)) + { + target.Add(value.Trim()); + } + } +} + internal sealed record ExcititorTimelineEvent( string Type, string Tenant, diff --git a/src/Concelier/StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj b/src/Concelier/StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj index fdae87e49..433beda14 100644 --- a/src/Concelier/StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj +++ b/src/Concelier/StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj @@ -25,7 +25,13 @@ + + + + + + diff --git a/src/Concelier/StellaOps.Excititor.WebService/TASKS.md b/src/Concelier/StellaOps.Excititor.WebService/TASKS.md index 8e86e80bb..b5d4bf485 100644 --- a/src/Concelier/StellaOps.Excititor.WebService/TASKS.md +++ b/src/Concelier/StellaOps.Excititor.WebService/TASKS.md @@ -11,3 +11,4 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | NOMOCK-012 | DONE | 2026-04-14: Removed live `InMemoryVexProviderStore`, `InMemoryVexConnectorStateRepository`, and `InMemoryVexClaimStore` fallbacks so the web runtime now resolves the persistence-backed Excititor stores. | | REALPLAN-007-A | DONE | 2026-04-15: Removed the host-level `InMemoryVexAttestationStore` fallback; WebService now relies on the Postgres attestation store from `AddExcititorPersistence`, backed by durable attestation migration proof. | | REALPLAN-007-B | DONE | 2026-04-15: Removed the live `InMemoryGraphOverlayStore` fallback; WebService now binds `IGraphOverlayStore` to `PostgresGraphOverlayStore` and startup migrations create `vex.graph_overlays`. | +| REALPLAN-007-C | DONE | 2026-04-20: Removed the live Excititor verification fallback path so enabled signature verification now requires a real IssuerDirectory HTTP endpoint and only uses Valkey when explicitly configured; no process-local issuer/cache fallback remains in the default runtime wiring. | diff --git a/src/Concelier/StellaOps.Excititor.Worker/Options/BuiltInVexProviderDefaults.cs b/src/Concelier/StellaOps.Excititor.Worker/Options/BuiltInVexProviderDefaults.cs new file mode 100644 index 000000000..a9a1036f6 --- /dev/null +++ b/src/Concelier/StellaOps.Excititor.Worker/Options/BuiltInVexProviderDefaults.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace StellaOps.Excititor.Worker.Options; + +public static class BuiltInVexProviderDefaults +{ + private static readonly (string ProviderId, TimeSpan InitialDelay)[] PublicDefaults = + { + ("excititor:redhat", TimeSpan.FromMinutes(5)), + ("excititor:ubuntu", TimeSpan.FromMinutes(7)), + ("excititor:oracle", TimeSpan.FromMinutes(9)), + ("excititor:cisco", TimeSpan.FromMinutes(11)), + }; + + public static void SeedPublicDefaults(IList providers) + { + ArgumentNullException.ThrowIfNull(providers); + + if (providers.Count > 0) + { + return; + } + + foreach (var (providerId, initialDelay) in PublicDefaults) + { + providers.Add(new VexWorkerProviderOptions + { + ProviderId = providerId, + InitialDelay = initialDelay, + }); + } + } + + public static IReadOnlyList GetPublicProviderIds() + => PublicDefaults.Select(static item => item.ProviderId).ToArray(); +} diff --git a/src/Concelier/StellaOps.Excititor.Worker/Orchestration/VexWorkerOrchestratorClient.cs b/src/Concelier/StellaOps.Excititor.Worker/Orchestration/VexWorkerOrchestratorClient.cs index b0b64b84f..b8ed3de42 100644 --- a/src/Concelier/StellaOps.Excititor.Worker/Orchestration/VexWorkerOrchestratorClient.cs +++ b/src/Concelier/StellaOps.Excititor.Worker/Orchestration/VexWorkerOrchestratorClient.cs @@ -243,7 +243,6 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient var updated = state with { - LastUpdated = result.CompletedAt, LastSuccessAt = result.CompletedAt, LastHeartbeatAt = result.CompletedAt, LastHeartbeatStatus = VexWorkerHeartbeatStatus.Succeeded.ToString(), diff --git a/src/Concelier/StellaOps.Excititor.Worker/Program.cs b/src/Concelier/StellaOps.Excititor.Worker/Program.cs index b74b83825..31292578a 100644 --- a/src/Concelier/StellaOps.Excititor.Worker/Program.cs +++ b/src/Concelier/StellaOps.Excititor.Worker/Program.cs @@ -7,6 +7,13 @@ using Microsoft.Extensions.Options; using StellaOps.Excititor.Attestation.Extensions; using StellaOps.Excititor.Attestation.Verification; using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.Cisco.CSAF.DependencyInjection; +using StellaOps.Excititor.Connectors.MSRC.CSAF.DependencyInjection; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.DependencyInjection; +using StellaOps.Excititor.Connectors.Oracle.CSAF.DependencyInjection; +using StellaOps.Excititor.Connectors.RedHat.CSAF.DependencyInjection; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.DependencyInjection; +using StellaOps.Excititor.Connectors.Ubuntu.CSAF.DependencyInjection; using StellaOps.Excititor.Core; using StellaOps.Excititor.Core.Aoc; using StellaOps.Excititor.Core.Orchestration; @@ -46,14 +53,25 @@ services.PostConfigure(options => options.Refresh.Enabled = false; } }); -// VEX connectors are loaded via plugin catalog below -// Direct connector registration removed in favor of plugin-based loading +// Register built-in connectors directly so document defaults work even when no plugin bundle is mounted. +services.AddRedHatCsafConnector(); +services.AddUbuntuCsafConnector(); +services.AddOracleCsafConnector(); +services.AddCiscoCsafConnector(); +services.AddRancherHubConnector(); +services.AddOciOpenVexAttestationConnector(); +var msrcConnectorSection = configuration.GetSection("Excititor").GetSection("Connectors").GetSection("Msrc"); +if (msrcConnectorSection.Exists()) +{ + services.AddMsrcCsafConnector(options => msrcConnectorSection.Bind(options)); +} services.AddOptions() .Bind(configuration.GetSection("Excititor:Storage")) .ValidateOnStart(); services.AddExcititorPersistence(configuration); +services.AddVexNormalization(); services.AddCsafNormalizer(); services.AddCycloneDxNormalizer(); services.AddOpenVexNormalizer(); @@ -85,13 +103,7 @@ services.AddSingleton(TimeProvider.System); services.TryAddSingleton(); services.PostConfigure(options => { - if (!options.Providers.Any(provider => string.Equals(provider.ProviderId, "excititor:redhat", StringComparison.OrdinalIgnoreCase))) - { - options.Providers.Add(new VexWorkerProviderOptions - { - ProviderId = "excititor:redhat", - }); - } + BuiltInVexProviderDefaults.SeedPublicDefaults(options.Providers); }); // Load VEX connector plugins services.AddSingleton(); diff --git a/src/Concelier/StellaOps.Excititor.Worker/Scheduling/DefaultVexProviderRunner.cs b/src/Concelier/StellaOps.Excititor.Worker/Scheduling/DefaultVexProviderRunner.cs index 1045fd406..9fcf6a2ba 100644 --- a/src/Concelier/StellaOps.Excititor.Worker/Scheduling/DefaultVexProviderRunner.cs +++ b/src/Concelier/StellaOps.Excititor.Worker/Scheduling/DefaultVexProviderRunner.cs @@ -126,8 +126,11 @@ internal sealed class DefaultVexProviderRunner : IVexProviderRunner var verifyingSink = new VerifyingVexRawDocumentSink(rawStore, signatureVerifier); + var hasResumeState = stateBeforeRun is not null && + (stateBeforeRun.DocumentDigests.Length > 0 || stateBeforeRun.ResumeTokens.Count > 0); + var connectorContext = new VexConnectorContext( - Since: stateBeforeRun?.LastUpdated, + Since: hasResumeState ? stateBeforeRun!.LastUpdated : null, Settings: effectiveSettings, RawSink: verifyingSink, SignatureVerifier: signatureVerifier, diff --git a/src/Concelier/StellaOps.Excititor.Worker/StellaOps.Excititor.Worker.csproj b/src/Concelier/StellaOps.Excititor.Worker/StellaOps.Excititor.Worker.csproj index 656c764dd..dd2ef2b9c 100644 --- a/src/Concelier/StellaOps.Excititor.Worker/StellaOps.Excititor.Worker.csproj +++ b/src/Concelier/StellaOps.Excititor.Worker/StellaOps.Excititor.Worker.csproj @@ -13,7 +13,13 @@ + + + + + + diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AGENTS.md b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AGENTS.md index 2547494d6..02fbd1fde 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AGENTS.md +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AGENTS.md @@ -16,7 +16,7 @@ Bootstrap the ACSC (Australian Cyber Security Centre) advisories connector so th - `Concelier.Testing` for integration harnesses and snapshot helpers. ## Interfaces & Contracts -- Job kinds should follow the pattern `acsc:fetch`, `acsc:parse`, `acsc:map`. +- Job kinds should follow the pattern `source:auscert:fetch`, `source:auscert:parse`, `source:auscert:map` (with `source:auscert:probe` for reachability checks). - Documents persisted to PostgreSQL must include ETag/Last-Modified metadata when the source exposes it. - Canonical advisories must emit aliases (ACSC ID + CVE IDs) and references (official bulletin + vendor notices). diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AcscConnectorPlugin.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AcscConnectorPlugin.cs index 768c8c31a..fba7d93ca 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AcscConnectorPlugin.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AcscConnectorPlugin.cs @@ -5,7 +5,7 @@ namespace StellaOps.Concelier.Connector.Acsc; public sealed class AcscConnectorPlugin : IConnectorPlugin { - public const string SourceName = "acsc"; + public const string SourceName = "auscert"; public string Name => SourceName; diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AcscDependencyInjectionRoutine.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AcscDependencyInjectionRoutine.cs index afb8bf225..25354b107 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AcscDependencyInjectionRoutine.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AcscDependencyInjectionRoutine.cs @@ -9,7 +9,8 @@ namespace StellaOps.Concelier.Connector.Acsc; public sealed class AcscDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "concelier:sources:acsc"; + private const string ConfigurationSection = "concelier:sources:auscert"; + private const string LegacyConfigurationSection = "concelier:sources:acsc"; private const string FetchCron = "7,37 * * * *"; private const string ParseCron = "12,42 * * * *"; @@ -29,6 +30,7 @@ public sealed class AcscDependencyInjectionRoutine : IDependencyInjectionRoutine services.AddAcscConnector(options => { + configuration.GetSection(LegacyConfigurationSection).Bind(options); configuration.GetSection(ConfigurationSection).Bind(options); options.Validate(); }); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Jobs.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Jobs.cs index 9ed3d62d5..e12079a98 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Jobs.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Jobs.cs @@ -4,10 +4,10 @@ namespace StellaOps.Concelier.Connector.Acsc; internal static class AcscJobKinds { - public const string Fetch = "source:acsc:fetch"; - public const string Parse = "source:acsc:parse"; - public const string Map = "source:acsc:map"; - public const string Probe = "source:acsc:probe"; + public const string Fetch = "source:auscert:fetch"; + public const string Parse = "source:auscert:parse"; + public const string Map = "source:auscert:map"; + public const string Probe = "source:auscert:probe"; } internal sealed class AcscFetchJob : IJob diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/README.md b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/README.md index 430cbf8d3..5d13dcb79 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/README.md +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/README.md @@ -3,7 +3,7 @@ Australian Cyber Security Centre (ACSC) connector that ingests RSS/Atom advisories, sanitises embedded HTML, and maps entries into canonical `Advisory` records for Concelier. ### Configuration -Settings live under `concelier:sources:acsc` (see `AcscOptions`): +Settings live under `concelier:sources:auscert` (see `AcscOptions`). Legacy `concelier:sources:acsc` binding is still accepted for compatibility: | Setting | Description | Default | | --- | --- | --- | @@ -23,10 +23,10 @@ The dependency injection routine registers the connector plus scheduled jobs: | Job | Cron | Purpose | | --- | --- | --- | -| `source:acsc:fetch` | `7,37 * * * *` | Fetch RSS/Atom feeds (direct + relay fallback). | -| `source:acsc:parse` | `12,42 * * * *` | Persist sanitised DTOs (`acsc.feed.v1`). | -| `source:acsc:map` | `17,47 * * * *` | Map DTO entries into canonical advisories. | -| `source:acsc:probe` | `25,55 * * * *` | Verify direct endpoint health and adjust cursor preference. | +| `source:auscert:fetch` | `7,37 * * * *` | Fetch RSS/Atom feeds (direct + relay fallback). | +| `source:auscert:parse` | `12,42 * * * *` | Persist sanitised DTOs (`acsc.feed.v1`). | +| `source:auscert:map` | `17,47 * * * *` | Map DTO entries into canonical advisories. | +| `source:auscert:probe` | `25,55 * * * *` | Verify direct endpoint health and adjust cursor preference. | ### Metrics Emitted via `AcscDiagnostics` (`Meter` = `StellaOps.Concelier.Connector.Acsc`): diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/AGENTS.md b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/AGENTS.md index 1e4d22468..201062696 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/AGENTS.md +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/AGENTS.md @@ -16,7 +16,7 @@ Build the CCCS (Canadian Centre for Cyber Security) advisories connector so Conc - `Concelier.Testing` (integration fixtures and snapshot utilities). ## Interfaces & Contracts -- Job kinds: `cccs:fetch`, `cccs:parse`, `cccs:map`. +- Job kinds: `source:cccs:fetch`, `source:cccs:parse`, `source:cccs:map`. - Persist ETag/Last-Modified metadata when the upstream supports it. - Include alias entries for CCCS advisory IDs plus referenced CVE IDs. diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/CccsDependencyInjectionRoutine.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/CccsDependencyInjectionRoutine.cs index f6ec87c0b..cbb08dbab 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/CccsDependencyInjectionRoutine.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/CccsDependencyInjectionRoutine.cs @@ -24,10 +24,14 @@ public sealed class CccsDependencyInjectionRoutine : IDependencyInjectionRoutine }); services.AddTransient(); + services.AddTransient(); + services.AddTransient(); services.PostConfigure(options => { EnsureJob(options, CccsJobKinds.Fetch, typeof(CccsFetchJob)); + EnsureJob(options, CccsJobKinds.Parse, typeof(CccsParseJob)); + EnsureJob(options, CccsJobKinds.Map, typeof(CccsMapJob)); }); return services; diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Jobs.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Jobs.cs index 79257abeb..7d046e86e 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Jobs.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Jobs.cs @@ -9,6 +9,8 @@ namespace StellaOps.Concelier.Connector.Cccs; internal static class CccsJobKinds { public const string Fetch = "source:cccs:fetch"; + public const string Parse = "source:cccs:parse"; + public const string Map = "source:cccs:map"; } internal sealed class CccsFetchJob : IJob @@ -21,3 +23,25 @@ internal sealed class CccsFetchJob : IJob public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) => _connector.FetchAsync(context.Services, cancellationToken); } + +internal sealed class CccsParseJob : IJob +{ + private readonly CccsConnector _connector; + + public CccsParseJob(CccsConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class CccsMapJob : IJob +{ + private readonly CccsConnector _connector; + + public CccsMapJob(CccsConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/TASKS.md b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/TASKS.md index 00e41888c..814d1731c 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/TASKS.md +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/TASKS.md @@ -5,6 +5,7 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | Task ID | Status | Notes | | --- | --- | --- | +| CONN-ALIGN-005 | DONE | 2026-04-22 runtime-alignment sprint exposed host fetch/parse/map registration for `source:cccs:*`. | | AUDIT-0149-M | DONE | Revalidated 2026-01-06. | | AUDIT-0149-T | DONE | Revalidated 2026-01-06. | | AUDIT-0149-A | DONE | Revalidated 2026-01-06 (no changes). | diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/CertBundConnectorPlugin.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/CertBundConnectorPlugin.cs index 15f7c1b92..fdf204430 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/CertBundConnectorPlugin.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/CertBundConnectorPlugin.cs @@ -7,7 +7,7 @@ namespace StellaOps.Concelier.Connector.CertBund; public sealed class CertBundConnectorPlugin : IConnectorPlugin { - public const string SourceName = "cert-bund"; + public const string SourceName = "cert-de"; public string Name => SourceName; diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/CertBundDependencyInjectionRoutine.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/CertBundDependencyInjectionRoutine.cs index c1f07575f..295325d53 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/CertBundDependencyInjectionRoutine.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/CertBundDependencyInjectionRoutine.cs @@ -10,7 +10,8 @@ namespace StellaOps.Concelier.Connector.CertBund; public sealed class CertBundDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "concelier:sources:cert-bund"; + private const string ConfigurationSection = "concelier:sources:cert-de"; + private const string LegacyConfigurationSection = "concelier:sources:cert-bund"; public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) { @@ -19,6 +20,7 @@ public sealed class CertBundDependencyInjectionRoutine : IDependencyInjectionRou services.AddCertBundConnector(options => { + configuration.GetSection(LegacyConfigurationSection).Bind(options); configuration.GetSection(ConfigurationSection).Bind(options); options.Validate(); }); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/Jobs.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/Jobs.cs index baec6ee9f..edceb9a41 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/Jobs.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/Jobs.cs @@ -8,7 +8,7 @@ namespace StellaOps.Concelier.Connector.CertBund; internal static class CertBundJobKinds { - public const string Fetch = "source:cert-bund:fetch"; + public const string Fetch = "source:cert-de:fetch"; } internal sealed class CertBundFetchJob : IJob diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/AGENTS.md b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/AGENTS.md index 4e4844b3f..b4ab879cb 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/AGENTS.md +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/AGENTS.md @@ -16,7 +16,7 @@ Implement the CERT/CC (Carnegie Mellon CERT Coordination Center) advisory connec - `Concelier.Testing` (integration tests and snapshots). ## Interfaces & Contracts -- Job kinds: `certcc:fetch`, `certcc:parse`, `certcc:map`. +- Job kinds: `source:cert-cc:fetch`, `source:cert-cc:parse`, `source:cert-cc:map`. - Persist upstream caching metadata (ETag/Last-Modified) when available. - Aliases should capture CERT/CC VU IDs and referenced CVEs. diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/CertCcDependencyInjectionRoutine.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/CertCcDependencyInjectionRoutine.cs index ae5d980c4..bee70ef7b 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/CertCcDependencyInjectionRoutine.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/CertCcDependencyInjectionRoutine.cs @@ -24,10 +24,14 @@ public sealed class CertCcDependencyInjectionRoutine : IDependencyInjectionRouti }); services.AddTransient(); + services.AddTransient(); + services.AddTransient(); services.PostConfigure(options => { EnsureJob(options, CertCcJobKinds.Fetch, typeof(CertCcFetchJob)); + EnsureJob(options, CertCcJobKinds.Parse, typeof(CertCcParseJob)); + EnsureJob(options, CertCcJobKinds.Map, typeof(CertCcMapJob)); }); return services; diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Jobs.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Jobs.cs index d598c148a..d524c7aad 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Jobs.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Jobs.cs @@ -9,6 +9,8 @@ namespace StellaOps.Concelier.Connector.CertCc; internal static class CertCcJobKinds { public const string Fetch = "source:cert-cc:fetch"; + public const string Parse = "source:cert-cc:parse"; + public const string Map = "source:cert-cc:map"; } internal sealed class CertCcFetchJob : IJob @@ -21,3 +23,25 @@ internal sealed class CertCcFetchJob : IJob public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) => _connector.FetchAsync(context.Services, cancellationToken); } + +internal sealed class CertCcParseJob : IJob +{ + private readonly CertCcConnector _connector; + + public CertCcParseJob(CertCcConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class CertCcMapJob : IJob +{ + private readonly CertCcConnector _connector; + + public CertCcMapJob(CertCcConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/TASKS.md b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/TASKS.md index 924a0217f..7f32440bb 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/TASKS.md +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/TASKS.md @@ -5,6 +5,7 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | Task ID | Status | Notes | | --- | --- | --- | +| CONN-ALIGN-005 | DONE | 2026-04-22 runtime-alignment sprint exposed host fetch/parse/map registration for `source:cert-cc:*`. | | AUDIT-0153-M | DONE | Revalidated 2026-01-06; no new findings. | | AUDIT-0153-T | DONE | Revalidated 2026-01-06; no new findings. | | AUDIT-0153-A | DONE | Applied fixes already in place; revalidated 2026-01-06 (no changes). | diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/Configuration/CertFrOptions.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/Configuration/CertFrOptions.cs index fa6acea23..f76210f98 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/Configuration/CertFrOptions.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/Configuration/CertFrOptions.cs @@ -6,7 +6,7 @@ public sealed class CertFrOptions { public const string HttpClientName = "cert-fr"; - public Uri FeedUri { get; set; } = new("https://www.cert.ssi.gouv.fr/feed/alertes/"); + public Uri FeedUri { get; set; } = new("https://www.cert.ssi.gouv.fr/alerte/feed/"); public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ghsa/GhsaConnector.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ghsa/GhsaConnector.cs index 4a637a5bf..d2bd40543 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ghsa/GhsaConnector.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ghsa/GhsaConnector.cs @@ -35,7 +35,7 @@ public sealed class GhsaConnector : IFeedConnector private readonly IDtoStore _dtoStore; private readonly IAdvisoryStore _advisoryStore; private readonly ISourceStateRepository _stateRepository; - private readonly GhsaOptions _options; + private readonly IOptionsMonitor _options; private readonly GhsaDiagnostics _diagnostics; private readonly TimeProvider _timeProvider; private readonly ILogger _logger; @@ -51,7 +51,7 @@ public sealed class GhsaConnector : IFeedConnector IDtoStore dtoStore, IAdvisoryStore advisoryStore, ISourceStateRepository stateRepository, - IOptions options, + IOptionsMonitor options, GhsaDiagnostics diagnostics, TimeProvider? timeProvider, ILogger logger, @@ -64,8 +64,7 @@ public sealed class GhsaConnector : IFeedConnector _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); - _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); - _options.Validate(); + _options = options ?? throw new ArgumentNullException(nameof(options)); _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); _timeProvider = timeProvider ?? TimeProvider.System; _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -79,13 +78,14 @@ public sealed class GhsaConnector : IFeedConnector { ArgumentNullException.ThrowIfNull(services); + var options = GetOptions(); var now = _timeProvider.GetUtcNow(); var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); var pendingDocuments = cursor.PendingDocuments.ToHashSet(); var pendingMappings = cursor.PendingMappings.ToHashSet(); - var since = cursor.CurrentWindowStart ?? cursor.LastUpdatedExclusive ?? now - _options.InitialBackfill; + var since = cursor.CurrentWindowStart ?? cursor.LastUpdatedExclusive ?? now - options.InitialBackfill; if (since > now) { since = now; @@ -103,17 +103,17 @@ public sealed class GhsaConnector : IFeedConnector var rateLimitHit = false; DateTimeOffset? maxUpdated = cursor.LastUpdatedExclusive; - while (hasMore && pagesFetched < _options.MaxPagesPerFetch) + while (hasMore && pagesFetched < options.MaxPagesPerFetch) { cancellationToken.ThrowIfCancellationRequested(); - var listUri = BuildListUri(since, until, page, _options.PageSize); + var listUri = BuildListUri(since, until, page, options.PageSize); var metadata = new Dictionary(StringComparer.Ordinal) { ["since"] = since.ToString("O", CultureInfo.InvariantCulture), ["until"] = until.ToString("O", CultureInfo.InvariantCulture), ["page"] = page.ToString(CultureInfo.InvariantCulture), - ["pageSize"] = _options.PageSize.ToString(CultureInfo.InvariantCulture), + ["pageSize"] = options.PageSize.ToString(CultureInfo.InvariantCulture), }; SourceFetchContentResult listResult; @@ -134,7 +134,7 @@ public sealed class GhsaConnector : IFeedConnector catch (HttpRequestException ex) { _diagnostics.FetchFailure(); - await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); + await _stateRepository.MarkFailureAsync(SourceName, now, options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); throw; } @@ -157,7 +157,7 @@ public sealed class GhsaConnector : IFeedConnector break; } - var pageModel = GhsaListParser.Parse(listResult.Content, page, _options.PageSize); + var pageModel = GhsaListParser.Parse(listResult.Content, page, options.PageSize); if (pageModel.Items.Count == 0) { @@ -239,9 +239,9 @@ public sealed class GhsaConnector : IFeedConnector page = pageModel.NextPageCandidate; pagesFetched++; - if (!rateLimitHit && hasMore && _options.RequestDelay > TimeSpan.Zero) + if (!rateLimitHit && hasMore && options.RequestDelay > TimeSpan.Zero) { - await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + await Task.Delay(options.RequestDelay, cancellationToken).ConfigureAwait(false); } } @@ -457,7 +457,7 @@ public sealed class GhsaConnector : IFeedConnector } var key = (snapshot.Phase, snapshot.Resource ?? "global"); - var warn = snapshot.Remaining.Value <= _options.RateLimitWarningThreshold; + var warn = snapshot.Remaining.Value <= GetOptions().RateLimitWarningThreshold; lock (_rateLimitWarningLock) { @@ -543,7 +543,7 @@ public sealed class GhsaConnector : IFeedConnector if (snapshot.Value.Remaining.HasValue && snapshot.Value.Remaining.Value <= 0) { _diagnostics.RateLimitExhausted(phase); - var delay = snapshot.Value.RetryAfter ?? snapshot.Value.ResetAfter ?? _options.SecondaryRateLimitBackoff; + var delay = snapshot.Value.RetryAfter ?? snapshot.Value.ResetAfter ?? GetOptions().SecondaryRateLimitBackoff; if (delay > TimeSpan.Zero) { @@ -562,6 +562,13 @@ public sealed class GhsaConnector : IFeedConnector return false; } + private GhsaOptions GetOptions() + { + var options = _options.CurrentValue; + options.Validate(); + return options; + } + /// /// Ingests GHSA advisory to canonical advisory service for deduplication. /// Creates one RawAdvisory per affected package. diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ghsa/GhsaDependencyInjectionRoutine.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ghsa/GhsaDependencyInjectionRoutine.cs index 19969ddef..fa02d0244 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ghsa/GhsaDependencyInjectionRoutine.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ghsa/GhsaDependencyInjectionRoutine.cs @@ -28,7 +28,6 @@ public sealed class GhsaDependencyInjectionRoutine : IDependencyInjectionRoutine services.AddGhsaConnector(options => { configuration.GetSection(ConfigurationSection).Bind(options); - options.Validate(); }); var scheduler = new JobSchedulerBuilder(services); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ghsa/GhsaServiceCollectionExtensions.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ghsa/GhsaServiceCollectionExtensions.cs index 6fe9b0123..af26d4714 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ghsa/GhsaServiceCollectionExtensions.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ghsa/GhsaServiceCollectionExtensions.cs @@ -14,12 +14,11 @@ public static class GhsaServiceCollectionExtensions ArgumentNullException.ThrowIfNull(configure); services.AddOptions() - .Configure(configure) - .PostConfigure(static opts => opts.Validate()); + .Configure(configure); services.AddSourceHttpClient(GhsaOptions.HttpClientName, (sp, clientOptions) => { - var options = sp.GetRequiredService>().Value; + var options = sp.GetRequiredService>().CurrentValue; clientOptions.BaseAddress = options.BaseEndpoint; clientOptions.Timeout = TimeSpan.FromSeconds(30); clientOptions.UserAgent = "StellaOps.Concelier.Ghsa/1.0"; diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Cisa/AGENTS.md b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Cisa/AGENTS.md index d181b9ce4..64005af93 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Cisa/AGENTS.md +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Cisa/AGENTS.md @@ -16,7 +16,7 @@ Implement the CISA ICS advisory connector to ingest US CISA Industrial Control S - `Concelier.Testing` (integration fixtures and snapshots). ## Interfaces & Contracts -- Job kinds: `ics-cisa:fetch`, `ics-cisa:parse`, `ics-cisa:map`. +- Job kinds: `source:us-cert:fetch`, `source:us-cert:parse`, `source:us-cert:map`. - Persist upstream caching metadata (ETag/Last-Modified) when available. - Alias set should include CISA ICS advisory IDs and referenced CVE IDs. diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Cisa/IcsCisaDependencyInjectionRoutine.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Cisa/IcsCisaDependencyInjectionRoutine.cs index bb4a4e8c5..1bf60fa47 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Cisa/IcsCisaDependencyInjectionRoutine.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Cisa/IcsCisaDependencyInjectionRoutine.cs @@ -10,7 +10,8 @@ namespace StellaOps.Concelier.Connector.Ics.Cisa; public sealed class IcsCisaDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "concelier:sources:ics-cisa"; + private const string ConfigurationSection = "concelier:sources:us-cert"; + private const string LegacyConfigurationSection = "concelier:sources:ics-cisa"; public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) { @@ -19,6 +20,7 @@ public sealed class IcsCisaDependencyInjectionRoutine : IDependencyInjectionRout services.AddIcsCisaConnector(options => { + configuration.GetSection(LegacyConfigurationSection).Bind(options); configuration.GetSection(ConfigurationSection).Bind(options); options.Validate(); }); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Cisa/Jobs.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Cisa/Jobs.cs index cc7dc5e5c..2ba947eeb 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Cisa/Jobs.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Cisa/Jobs.cs @@ -8,9 +8,9 @@ namespace StellaOps.Concelier.Connector.Ics.Cisa; internal static class IcsCisaJobKinds { - public const string Fetch = "source:ics-cisa:fetch"; - public const string Parse = "source:ics-cisa:parse"; - public const string Map = "source:ics-cisa:map"; + public const string Fetch = "source:us-cert:fetch"; + public const string Parse = "source:us-cert:parse"; + public const string Map = "source:us-cert:map"; } internal sealed class IcsCisaFetchJob : IJob diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Kaspersky/AGENTS.md b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Kaspersky/AGENTS.md index 5a7880ce4..1af8a3419 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Kaspersky/AGENTS.md +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Kaspersky/AGENTS.md @@ -9,7 +9,7 @@ Kaspersky ICS-CERT connector; authoritative for OT/ICS vendor advisories covered - Source.Common (HTTP, HTML helpers, validators). - Storage.Postgres (document, dto, advisory, alias, affected, reference, source_state). - Models (canonical; affected.platform="ics-vendor", tags for device families). -- Core/WebService (jobs: source:ics-kaspersky:fetch|parse|map). +- Core/WebService (jobs: source:kaspersky-ics:fetch|parse|map). - Merge engine respects ICS vendor authority for OT impact. ## Interfaces & contracts - Aliases: CVE ids; if stable ICS-CERT advisory id exists, store scheme "ICS-KASP". @@ -20,7 +20,7 @@ Kaspersky ICS-CERT connector; authoritative for OT/ICS vendor advisories covered In: ICS advisory mapping, affected vendor products, mitigation references. Out: firmware downloads; reverse-engineering artifacts. ## Observability & security expectations -- Metrics: SourceDiagnostics publishes `concelier.source.http.*` counters/histograms with `concelier.source=ics-kaspersky` to track fetch totals, parse failures, and mapped affected counts. +- Metrics: SourceDiagnostics publishes `concelier.source.http.*` counters/histograms with `concelier.source=kaspersky-ics` to track fetch totals, parse failures, and mapped affected counts. - Logs: slugs, vendor/product counts, timing; allowlist host. ## Tests - Author and review coverage in `../StellaOps.Concelier.Connector.Ics.Kaspersky.Tests`. diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Kaspersky/Jobs.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Kaspersky/Jobs.cs index 145d34e8d..465e96988 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Kaspersky/Jobs.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Kaspersky/Jobs.cs @@ -8,9 +8,9 @@ namespace StellaOps.Concelier.Connector.Ics.Kaspersky; internal static class KasperskyJobKinds { - public const string Fetch = "source:ics-kaspersky:fetch"; - public const string Parse = "source:ics-kaspersky:parse"; - public const string Map = "source:ics-kaspersky:map"; + public const string Fetch = "source:kaspersky-ics:fetch"; + public const string Parse = "source:kaspersky-ics:parse"; + public const string Map = "source:kaspersky-ics:map"; } internal sealed class KasperskyFetchJob : IJob diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Kaspersky/KasperskyDependencyInjectionRoutine.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Kaspersky/KasperskyDependencyInjectionRoutine.cs index 0e9cf4720..08bb5fc5b 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Kaspersky/KasperskyDependencyInjectionRoutine.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ics.Kaspersky/KasperskyDependencyInjectionRoutine.cs @@ -10,7 +10,8 @@ namespace StellaOps.Concelier.Connector.Ics.Kaspersky; public sealed class KasperskyDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "concelier:sources:ics-kaspersky"; + private const string ConfigurationSection = "concelier:sources:kaspersky-ics"; + private const string LegacyConfigurationSection = "concelier:sources:ics-kaspersky"; public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) { @@ -19,6 +20,7 @@ public sealed class KasperskyDependencyInjectionRoutine : IDependencyInjectionRo services.AddKasperskyIcsConnector(options => { + configuration.GetSection(LegacyConfigurationSection).Bind(options); configuration.GetSection(ConfigurationSection).Bind(options); options.Validate(); }); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Jvn/AGENTS.md b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Jvn/AGENTS.md index 3f69e95b0..9083e61b1 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Jvn/AGENTS.md +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Jvn/AGENTS.md @@ -9,7 +9,7 @@ Japan JVN/MyJVN connector; national CERT enrichment with strong identifiers (JVN - Source.Common (HTTP, pagination, XML or XSD validators, retries/backoff). - Storage.Postgres (document, dto, advisory, alias, affected (when concrete), reference, jp_flags, source_state). - Models (canonical Advisory/Affected/Provenance). -- Core/WebService (jobs: source:jvn:fetch|parse|map). +- Core/WebService (jobs: source:jpcert:fetch|parse|map). - Merge engine applies enrichment precedence (does not override distro or PSIRT ranges unless JVN gives explicit package truth). ## Interfaces & contracts - Aliases include JVNDB-YYYY-NNNNN and CVE ids; scheme "JVNDB". @@ -21,7 +21,7 @@ Japan JVN/MyJVN connector; national CERT enrichment with strong identifiers (JVN In: JVN/MyJVN ingestion, aliases, jp_flags, enrichment mapping, watermarking. Out: overriding distro or PSIRT ranges without concrete evidence; scraping unofficial mirrors. ## Observability & security expectations -- Metrics: SourceDiagnostics emits `concelier.source.http.*` counters/histograms tagged `concelier.source=jvn`, enabling dashboards to track fetch requests, item counts, parse failures, and enrichment/map activity (including jp_flags) via tag filters. +- Metrics: SourceDiagnostics emits `concelier.source.http.*` counters/histograms tagged `concelier.source=jpcert`, enabling dashboards to track fetch requests, item counts, parse failures, and enrichment/map activity (including jp_flags) via tag filters. - Logs: window bounds, jvndb ids processed, vendor_status distribution; redact API keys. ## Tests - Author and review coverage in `../StellaOps.Concelier.Connector.Jvn.Tests`. diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Jvn/Jobs.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Jvn/Jobs.cs index 7ad321635..825a0400e 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Jvn/Jobs.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Jvn/Jobs.cs @@ -8,9 +8,9 @@ namespace StellaOps.Concelier.Connector.Jvn; internal static class JvnJobKinds { - public const string Fetch = "source:jvn:fetch"; - public const string Parse = "source:jvn:parse"; - public const string Map = "source:jvn:map"; + public const string Fetch = "source:jpcert:fetch"; + public const string Parse = "source:jpcert:parse"; + public const string Map = "source:jpcert:map"; } internal sealed class JvnFetchJob : IJob diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Jvn/JvnConnectorPlugin.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Jvn/JvnConnectorPlugin.cs index 56f7a2dee..9aa16d804 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Jvn/JvnConnectorPlugin.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Jvn/JvnConnectorPlugin.cs @@ -7,7 +7,7 @@ public sealed class JvnConnectorPlugin : IConnectorPlugin { public string Name => SourceName; - public static string SourceName => "jvn"; + public static string SourceName => "jpcert"; public bool IsAvailable(IServiceProvider services) => services is not null; diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Jvn/JvnDependencyInjectionRoutine.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Jvn/JvnDependencyInjectionRoutine.cs index dd7e9a687..11533dc91 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Jvn/JvnDependencyInjectionRoutine.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Jvn/JvnDependencyInjectionRoutine.cs @@ -10,7 +10,8 @@ namespace StellaOps.Concelier.Connector.Jvn; public sealed class JvnDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "concelier:sources:jvn"; + private const string ConfigurationSection = "concelier:sources:jpcert"; + private const string LegacyConfigurationSection = "concelier:sources:jvn"; public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) { @@ -19,6 +20,7 @@ public sealed class JvnDependencyInjectionRoutine : IDependencyInjectionRoutine services.AddJvnConnector(options => { + configuration.GetSection(LegacyConfigurationSection).Bind(options); configuration.GetSection(ConfigurationSection).Bind(options); options.Validate(); }); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kev/Jobs.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kev/Jobs.cs index 63e116297..53a912ec3 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kev/Jobs.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kev/Jobs.cs @@ -1,6 +1,7 @@ using StellaOps.Concelier.Core.Jobs; using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -11,6 +12,11 @@ internal static class KevJobKinds public const string Fetch = "source:kev:fetch"; public const string Parse = "source:kev:parse"; public const string Map = "source:kev:map"; + + public static bool ShouldForce(IReadOnlyDictionary parameters) + => parameters.TryGetValue("force", out var forceValue) + && forceValue is string force + && string.Equals(force, "true", StringComparison.OrdinalIgnoreCase); } internal sealed class KevFetchJob : IJob @@ -32,7 +38,7 @@ internal sealed class KevParseJob : IJob => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.ParseAsync(context.Services, cancellationToken); + => _connector.ParseAsync(context.Services, KevJobKinds.ShouldForce(context.Parameters), cancellationToken); } internal sealed class KevMapJob : IJob @@ -43,5 +49,5 @@ internal sealed class KevMapJob : IJob => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.MapAsync(context.Services, cancellationToken); + => _connector.MapAsync(context.Services, KevJobKinds.ShouldForce(context.Parameters), cancellationToken); } diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kev/KevConnector.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kev/KevConnector.cs index e8914b606..571895818 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kev/KevConnector.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kev/KevConnector.cs @@ -6,6 +6,7 @@ using StellaOps.Concelier.Connector.Common.Fetch; using StellaOps.Concelier.Connector.Common.Json; using StellaOps.Concelier.Connector.Kev.Configuration; using StellaOps.Concelier.Connector.Kev.Internal; +using StellaOps.Concelier.Core.Canonical; using StellaOps.Concelier.Documents; using StellaOps.Concelier.Models; using StellaOps.Concelier.Storage; @@ -44,6 +45,7 @@ public sealed class KevConnector : IFeedConnector private readonly IJsonSchemaValidator _schemaValidator; private readonly ICryptoHash _hash; private readonly TimeProvider _timeProvider; + private readonly ICanonicalAdvisoryService? _canonicalService; private readonly ILogger _logger; private readonly KevDiagnostics _diagnostics; @@ -59,7 +61,8 @@ public sealed class KevConnector : IFeedConnector ICryptoHash cryptoHash, KevDiagnostics diagnostics, TimeProvider? timeProvider, - ILogger logger) + ILogger logger, + ICanonicalAdvisoryService? canonicalService = null) { _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); @@ -74,6 +77,7 @@ public sealed class KevConnector : IFeedConnector _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); _timeProvider = timeProvider ?? TimeProvider.System; _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _canonicalService = canonicalService; } public string SourceName => KevConnectorPlugin.SourceName; @@ -161,12 +165,16 @@ public sealed class KevConnector : IFeedConnector } } - public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + => ParseAsync(services, force: false, cancellationToken); + + public async Task ParseAsync(IServiceProvider services, bool force, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(services); var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingDocuments.Count == 0) + var documentIds = await ResolveDocumentIdsAsync(cursor.PendingDocuments, force, "parse", cancellationToken).ConfigureAwait(false); + if (documentIds.Count == 0) { return; } @@ -176,7 +184,7 @@ public sealed class KevConnector : IFeedConnector var latestCatalogVersion = cursor.CatalogVersion; var latestCatalogReleased = cursor.CatalogReleased; - foreach (var documentId in cursor.PendingDocuments) + foreach (var documentId in documentIds) { cancellationToken.ThrowIfCancellationRequested(); @@ -311,12 +319,16 @@ public sealed class KevConnector : IFeedConnector await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); } - public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + => MapAsync(services, force: false, cancellationToken); + + public async Task MapAsync(IServiceProvider services, bool force, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(services); var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingMappings.Count == 0) + var documentIds = await ResolveDocumentIdsAsync(cursor.PendingMappings, force, "map", cancellationToken).ConfigureAwait(false); + if (documentIds.Count == 0) { return; } @@ -330,7 +342,7 @@ public sealed class KevConnector : IFeedConnector pendingDocuments.Remove(documentId); } - foreach (var documentId in cursor.PendingMappings) + foreach (var documentId in documentIds) { cancellationToken.ThrowIfCancellationRequested(); @@ -344,9 +356,10 @@ public sealed class KevConnector : IFeedConnector } KevCatalogDto? catalog; + string dtoJson; try { - var dtoJson = dtoRecord.Payload.ToJson(new StellaOps.Concelier.Documents.IO.JsonWriterSettings + dtoJson = dtoRecord.Payload.ToJson(new StellaOps.Concelier.Documents.IO.JsonWriterSettings { OutputMode = StellaOps.Concelier.Documents.IO.JsonOutputMode.RelaxedExtendedJson, }); @@ -385,6 +398,7 @@ public sealed class KevConnector : IFeedConnector foreach (var advisory in advisories) { await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await IngestToCanonicalAsync(advisory, dtoJson, document.FetchedAt, cancellationToken).ConfigureAwait(false); } await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); @@ -397,6 +411,44 @@ public sealed class KevConnector : IFeedConnector await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); } + private async Task> ResolveDocumentIdsAsync( + IReadOnlyCollection pendingDocumentIds, + bool force, + string operation, + CancellationToken cancellationToken) + { + if (pendingDocumentIds.Count > 0) + { + return pendingDocumentIds.ToArray(); + } + + if (!force) + { + return Array.Empty(); + } + + var latestDocument = await _documentStore + .FindBySourceAndUriAsync(SourceName, _options.FeedUri.ToString(), cancellationToken) + .ConfigureAwait(false); + + if (latestDocument is null) + { + _logger.LogWarning( + "KEV {Operation}: force replay requested but no stored catalog document exists for {FeedUri}", + operation, + _options.FeedUri); + return Array.Empty(); + } + + _logger.LogInformation( + "KEV {Operation}: force replay requested for cached document {DocumentId} (status={Status})", + operation, + latestDocument.Id, + latestDocument.Status); + + return new[] { latestDocument.Id }; + } + private async Task GetCursorAsync(CancellationToken cancellationToken) { var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); @@ -453,6 +505,83 @@ public sealed class KevConnector : IFeedConnector private static Uri? TryParseUri(string? value) => Uri.TryCreate(value, UriKind.Absolute, out var uri) ? uri : null; + private async Task IngestToCanonicalAsync( + Advisory advisory, + string rawPayloadJson, + DateTimeOffset fetchedAt, + CancellationToken cancellationToken) + { + if (_canonicalService is null || advisory.AffectedPackages.IsEmpty) + { + return; + } + + var cve = advisory.Aliases + .FirstOrDefault(static alias => alias.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase)) + ?? advisory.AdvisoryKey; + + var weaknesses = advisory.Cwes + .Where(static weakness => weakness.Identifier.StartsWith("CWE-", StringComparison.OrdinalIgnoreCase)) + .Select(static weakness => weakness.Identifier) + .ToList(); + + foreach (var affected in advisory.AffectedPackages) + { + if (string.IsNullOrWhiteSpace(affected.Identifier)) + { + continue; + } + + string? versionRangeJson = null; + if (!affected.VersionRanges.IsEmpty) + { + var firstRange = affected.VersionRanges[0]; + var rangeObject = new + { + introduced = firstRange.IntroducedVersion, + @fixed = firstRange.FixedVersion, + last_affected = firstRange.LastAffectedVersion + }; + versionRangeJson = JsonSerializer.Serialize(rangeObject); + } + + var rawAdvisory = new RawAdvisory + { + SourceAdvisoryId = advisory.AdvisoryKey, + Cve = cve, + AffectsKey = affected.Identifier, + VersionRangeJson = versionRangeJson, + Weaknesses = weaknesses, + PatchLineage = null, + Severity = advisory.Severity, + Title = advisory.Title, + Summary = advisory.Summary, + VendorStatus = VendorStatus.Affected, + RawPayloadJson = rawPayloadJson, + FetchedAt = fetchedAt + }; + + try + { + var result = await _canonicalService.IngestAsync(SourceName, rawAdvisory, cancellationToken).ConfigureAwait(false); + + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug( + "Canonical ingest for {AdvisoryKey}/{AffectsKey}: {Decision} (canonical={CanonicalId})", + advisory.AdvisoryKey, affected.Identifier, result.Decision, result.CanonicalId); + } + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "Failed to ingest {AdvisoryKey}/{AffectsKey} to canonical service", + advisory.AdvisoryKey, affected.Identifier); + } + } + } + private Guid ComputeDeterministicId(string source, string identifier) { var input = Encoding.UTF8.GetBytes($"{source}:{identifier}"); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kisa/AGENTS.md b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kisa/AGENTS.md index 779166d70..f20656384 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kisa/AGENTS.md +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kisa/AGENTS.md @@ -16,7 +16,7 @@ Deliver the KISA (Korea Internet & Security Agency) advisory connector to ingest - `Concelier.Testing` (integration fixtures and snapshots). ## Interfaces & Contracts -- Job kinds: `kisa:fetch`, `kisa:parse`, `kisa:map`. +- Job kinds: `source:krcert:fetch`, `source:krcert:parse`, `source:krcert:map`. - Persist upstream caching metadata (e.g., ETag/Last-Modified) when available. - Alias set should include KISA advisory identifiers and CVE IDs. diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kisa/Jobs.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kisa/Jobs.cs index 25d86e3bb..6221c66bd 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kisa/Jobs.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kisa/Jobs.cs @@ -8,7 +8,9 @@ namespace StellaOps.Concelier.Connector.Kisa; internal static class KisaJobKinds { - public const string Fetch = "source:kisa:fetch"; + public const string Fetch = "source:krcert:fetch"; + public const string Parse = "source:krcert:parse"; + public const string Map = "source:krcert:map"; } internal sealed class KisaFetchJob : IJob @@ -21,3 +23,25 @@ internal sealed class KisaFetchJob : IJob public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) => _connector.FetchAsync(context.Services, cancellationToken); } + +internal sealed class KisaParseJob : IJob +{ + private readonly KisaConnector _connector; + + public KisaParseJob(KisaConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class KisaMapJob : IJob +{ + private readonly KisaConnector _connector; + + public KisaMapJob(KisaConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kisa/KisaConnectorPlugin.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kisa/KisaConnectorPlugin.cs index 33348df2e..4a24919b8 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kisa/KisaConnectorPlugin.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kisa/KisaConnectorPlugin.cs @@ -7,7 +7,7 @@ namespace StellaOps.Concelier.Connector.Kisa; public sealed class KisaConnectorPlugin : IConnectorPlugin { - public const string SourceName = "kisa"; + public const string SourceName = "krcert"; public string Name => SourceName; diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kisa/KisaDependencyInjectionRoutine.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kisa/KisaDependencyInjectionRoutine.cs index 06c3d0fa6..1e0056e57 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kisa/KisaDependencyInjectionRoutine.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kisa/KisaDependencyInjectionRoutine.cs @@ -10,7 +10,8 @@ namespace StellaOps.Concelier.Connector.Kisa; public sealed class KisaDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "concelier:sources:kisa"; + private const string ConfigurationSection = "concelier:sources:krcert"; + private const string LegacyConfigurationSection = "concelier:sources:kisa"; public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) { @@ -19,15 +20,20 @@ public sealed class KisaDependencyInjectionRoutine : IDependencyInjectionRoutine services.AddKisaConnector(options => { + configuration.GetSection(LegacyConfigurationSection).Bind(options); configuration.GetSection(ConfigurationSection).Bind(options); options.Validate(); }); services.AddTransient(); + services.AddTransient(); + services.AddTransient(); services.PostConfigure(options => { EnsureJob(options, KisaJobKinds.Fetch, typeof(KisaFetchJob)); + EnsureJob(options, KisaJobKinds.Parse, typeof(KisaParseJob)); + EnsureJob(options, KisaJobKinds.Map, typeof(KisaMapJob)); }); return services; diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kisa/TASKS.md b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kisa/TASKS.md index 7d8edc481..7bce7d390 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kisa/TASKS.md +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Kisa/TASKS.md @@ -5,6 +5,7 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | Task ID | Status | Notes | | --- | --- | --- | +| CONN-ALIGN-005 | DONE | 2026-04-22 runtime-alignment sprint exposed host fetch/parse/map registration for `source:krcert:*`. | | AUDIT-0185-M | DONE | Revalidated 2026-01-06; open findings recorded in audit report. | | AUDIT-0185-T | DONE | Revalidated 2026-01-06; open findings recorded in audit report. | | AUDIT-0185-A | TODO | Revalidated 2026-01-06; open findings pending approval. | diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Nvd/Jobs.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Nvd/Jobs.cs new file mode 100644 index 000000000..fae4b1770 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Nvd/Jobs.cs @@ -0,0 +1,43 @@ +using StellaOps.Concelier.Core.Jobs; + +namespace StellaOps.Concelier.Connector.Nvd; + +internal static class NvdJobKinds +{ + public const string Fetch = "source:nvd:fetch"; + public const string Parse = "source:nvd:parse"; + public const string Map = "source:nvd:map"; +} + +internal sealed class NvdFetchJob : IJob +{ + private readonly NvdConnector connector; + + public NvdFetchJob(NvdConnector connector) + => this.connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class NvdParseJob : IJob +{ + private readonly NvdConnector connector; + + public NvdParseJob(NvdConnector connector) + => this.connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class NvdMapJob : IJob +{ + private readonly NvdConnector connector; + + public NvdMapJob(NvdConnector connector) + => this.connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Nvd/NvdDependencyInjectionRoutine.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Nvd/NvdDependencyInjectionRoutine.cs new file mode 100644 index 000000000..1e5c2c559 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Nvd/NvdDependencyInjectionRoutine.cs @@ -0,0 +1,53 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Concelier.Connector.Nvd.Configuration; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.DependencyInjection; + +namespace StellaOps.Concelier.Connector.Nvd; + +public sealed class NvdDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "concelier:sources:nvd"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddNvdConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, NvdJobKinds.Fetch, typeof(NvdFetchJob)); + EnsureJob(options, NvdJobKinds.Parse, typeof(NvdParseJob)); + EnsureJob(options, NvdJobKinds.Map, typeof(NvdMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/Configuration/OsvOptions.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/Configuration/OsvOptions.cs index 232549402..35dc5a238 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/Configuration/OsvOptions.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/Configuration/OsvOptions.cs @@ -8,7 +8,7 @@ public sealed class OsvOptions public Uri BaseUri { get; set; } = new("https://osv-vulnerabilities.storage.googleapis.com/", UriKind.Absolute); - public IReadOnlyList Ecosystems { get; set; } = new[] { "PyPI", "npm", "Maven", "Go", "crates" }; + public IReadOnlyList Ecosystems { get; set; } = new[] { "PyPI", "npm", "Maven", "Go", "crates.io" }; public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(14); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/OsvConnector.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/OsvConnector.cs index 3bf8fc3a3..ae07dea3b 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/OsvConnector.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/OsvConnector.cs @@ -365,9 +365,11 @@ public sealed class OsvConnector : IFeedConnector var processedUpdated = false; var newDocuments = 0; - var minimumModified = existingLastModified.HasValue - ? existingLastModified.Value - _options.ModifiedTolerance - : now - _options.InitialBackfill; + var minimumModified = ComputeMinimumModified( + existingLastModified, + now, + _options.ModifiedTolerance, + _options.InitialBackfill); ProvenanceDiagnostics.ReportResumeWindow(SourceName, minimumModified, _logger); @@ -412,12 +414,7 @@ public sealed class OsvConnector : IFeedConnector continue; } - if (existingLastModified.HasValue && modified < existingLastModified.Value - _options.ModifiedTolerance) - { - continue; - } - - if (modified < currentMaxModified - _options.ModifiedTolerance) + if (IsOutsideResumeWindow(modified, existingLastModified, currentMaxModified, _options.ModifiedTolerance)) { continue; } @@ -523,6 +520,43 @@ public sealed class OsvConnector : IFeedConnector return builder.Uri; } + internal static DateTimeOffset ComputeMinimumModified( + DateTimeOffset? existingLastModified, + DateTimeOffset now, + TimeSpan modifiedTolerance, + TimeSpan initialBackfill) + => existingLastModified.HasValue + ? SafeSubtract(existingLastModified.Value, modifiedTolerance) + : SafeSubtract(now, initialBackfill); + + internal static bool IsOutsideResumeWindow( + DateTimeOffset modified, + DateTimeOffset? existingLastModified, + DateTimeOffset currentMaxModified, + TimeSpan modifiedTolerance) + { + if (existingLastModified.HasValue && + modified < SafeSubtract(existingLastModified.Value, modifiedTolerance)) + { + return true; + } + + return currentMaxModified != DateTimeOffset.MinValue && + modified < SafeSubtract(currentMaxModified, modifiedTolerance); + } + + internal static DateTimeOffset SafeSubtract(DateTimeOffset value, TimeSpan delta) + { + try + { + return value - delta; + } + catch (ArgumentOutOfRangeException) + { + return DateTimeOffset.MinValue; + } + } + private static string BuildDocumentUri(string ecosystem, string vulnerabilityId) { var safeId = vulnerabilityId.Replace(' ', '-'); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/TASKS.md b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/TASKS.md index 713b37526..6c947051e 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/TASKS.md +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/TASKS.md @@ -5,6 +5,7 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | Task ID | Status | Notes | | --- | --- | --- | +| CONN-ALIGN-003 | DOING | 2026-04-21 runtime validation: correcting live OSV archive defaults so the default ecosystem set matches the upstream `crates.io/all.zip` export instead of the dead `crates/all.zip` path. | | AUDIT-0189-M | DONE | Revalidated 2026-01-06; open findings recorded in audit report. | | AUDIT-0189-T | DONE | Revalidated 2026-01-06; open findings recorded in audit report. | | AUDIT-0189-A | TODO | Revalidated 2026-01-06; open findings pending approval. | diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Bdu/AGENTS.md b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Bdu/AGENTS.md index c46e25830..db32de160 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Bdu/AGENTS.md +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Bdu/AGENTS.md @@ -16,7 +16,7 @@ Implement the Russian BDU (Vulnerability Database) connector to ingest advisorie - `Concelier.Testing` (integration harness, snapshot utilities). ## Interfaces & Contracts -- Job kinds: `bdu:fetch`, `bdu:parse`, `bdu:map`. +- Job kinds: `source:fstec-bdu:fetch`, `source:fstec-bdu:parse`, `source:fstec-bdu:map`. - Persist upstream metadata (e.g., record modification timestamp) to drive incremental updates. - Alias set should include BDU identifiers and CVE IDs when present. diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Bdu/Jobs.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Bdu/Jobs.cs index 1e8452c20..2ca481f28 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Bdu/Jobs.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Bdu/Jobs.cs @@ -4,9 +4,9 @@ namespace StellaOps.Concelier.Connector.Ru.Bdu; internal static class RuBduJobKinds { - public const string Fetch = "source:ru-bdu:fetch"; - public const string Parse = "source:ru-bdu:parse"; - public const string Map = "source:ru-bdu:map"; + public const string Fetch = "source:fstec-bdu:fetch"; + public const string Parse = "source:fstec-bdu:parse"; + public const string Map = "source:fstec-bdu:map"; } internal sealed class RuBduFetchJob : IJob diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Bdu/RuBduDependencyInjectionRoutine.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Bdu/RuBduDependencyInjectionRoutine.cs index aa7463d18..99ec40401 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Bdu/RuBduDependencyInjectionRoutine.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Bdu/RuBduDependencyInjectionRoutine.cs @@ -9,7 +9,8 @@ namespace StellaOps.Concelier.Connector.Ru.Bdu; public sealed class RuBduDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "concelier:sources:ru-bdu"; + private const string ConfigurationSection = "concelier:sources:fstec-bdu"; + private const string LegacyConfigurationSection = "concelier:sources:ru-bdu"; public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) { @@ -18,6 +19,7 @@ public sealed class RuBduDependencyInjectionRoutine : IDependencyInjectionRoutin services.AddRuBduConnector(options => { + configuration.GetSection(LegacyConfigurationSection).Bind(options); configuration.GetSection(ConfigurationSection).Bind(options); options.Validate(); }); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Nkcki/AGENTS.md b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Nkcki/AGENTS.md index 969041313..e67132f12 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Nkcki/AGENTS.md +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Nkcki/AGENTS.md @@ -16,7 +16,7 @@ Implement the Russian NKTsKI (formerly NKCKI) advisories connector to ingest NKT - `Concelier.Testing` (integration fixtures, snapshots). ## Interfaces & Contracts -- Job kinds: `nkcki:fetch`, `nkcki:parse`, `nkcki:map`. +- Job kinds: `source:nkcki:fetch`, `source:nkcki:parse`, `source:nkcki:map`. - Persist upstream modification metadata to support incremental updates. - Alias set should include NKTsKI advisory IDs and CVEs when present. diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Nkcki/Jobs.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Nkcki/Jobs.cs index fcf537d48..1a9582e1f 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Nkcki/Jobs.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Nkcki/Jobs.cs @@ -4,9 +4,9 @@ namespace StellaOps.Concelier.Connector.Ru.Nkcki; internal static class RuNkckiJobKinds { - public const string Fetch = "source:ru-nkcki:fetch"; - public const string Parse = "source:ru-nkcki:parse"; - public const string Map = "source:ru-nkcki:map"; + public const string Fetch = "source:nkcki:fetch"; + public const string Parse = "source:nkcki:parse"; + public const string Map = "source:nkcki:map"; } internal sealed class RuNkckiFetchJob : IJob diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Nkcki/RuNkckiDependencyInjectionRoutine.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Nkcki/RuNkckiDependencyInjectionRoutine.cs index e63abc344..61fd6527b 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Nkcki/RuNkckiDependencyInjectionRoutine.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ru.Nkcki/RuNkckiDependencyInjectionRoutine.cs @@ -9,7 +9,8 @@ namespace StellaOps.Concelier.Connector.Ru.Nkcki; public sealed class RuNkckiDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "concelier:sources:ru-nkcki"; + private const string ConfigurationSection = "concelier:sources:nkcki"; + private const string LegacyConfigurationSection = "concelier:sources:ru-nkcki"; public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) { @@ -18,6 +19,7 @@ public sealed class RuNkckiDependencyInjectionRoutine : IDependencyInjectionRout services.AddRuNkckiConnector(options => { + configuration.GetSection(LegacyConfigurationSection).Bind(options); configuration.GetSection(ConfigurationSection).Bind(options); options.Validate(); }); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Adobe/AdobeDependencyInjectionRoutine.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Adobe/AdobeDependencyInjectionRoutine.cs new file mode 100644 index 000000000..54b9270b1 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Adobe/AdobeDependencyInjectionRoutine.cs @@ -0,0 +1,57 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.DependencyInjection; +using System; + +namespace StellaOps.Concelier.Connector.Vndr.Adobe; + +public sealed class AdobeDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "concelier:sources:adobe"; + private const string LegacyConfigurationSection = "concelier:sources:vndr-adobe"; + private const string LegacyVendorConfigurationSection = "concelier:sources:vndr:adobe"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddAdobeConnector(options => + { + configuration.GetSection(LegacyVendorConfigurationSection).Bind(options); + configuration.GetSection(LegacyConfigurationSection).Bind(options); + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, AdobeJobKinds.Fetch, typeof(AdobeFetchJob)); + EnsureJob(options, AdobeJobKinds.Parse, typeof(AdobeParseJob)); + EnsureJob(options, AdobeJobKinds.Map, typeof(AdobeMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Adobe/Jobs.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Adobe/Jobs.cs new file mode 100644 index 000000000..eaf5f3bc8 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Adobe/Jobs.cs @@ -0,0 +1,46 @@ +using StellaOps.Concelier.Core.Jobs; +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Concelier.Connector.Vndr.Adobe; + +internal static class AdobeJobKinds +{ + public const string Fetch = "source:adobe:fetch"; + public const string Parse = "source:adobe:parse"; + public const string Map = "source:adobe:map"; +} + +internal sealed class AdobeFetchJob : IJob +{ + private readonly AdobeConnector _connector; + + public AdobeFetchJob(AdobeConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class AdobeParseJob : IJob +{ + private readonly AdobeConnector _connector; + + public AdobeParseJob(AdobeConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class AdobeMapJob : IJob +{ + private readonly AdobeConnector _connector; + + public AdobeMapJob(AdobeConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Adobe/TASKS.md b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Adobe/TASKS.md index 787547ffc..35c1fd83b 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Adobe/TASKS.md +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Adobe/TASKS.md @@ -5,6 +5,7 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | Task ID | Status | Notes | | --- | --- | --- | +| CONN-ALIGN-007 | DONE | 2026-04-22: Promoted Adobe into the canonical `adobe` catalog/runtime path with built-in host jobs, alias normalization, and updated operator runbooks. | | AUDIT-0197-M | DONE | Revalidated 2026-01-06; open findings recorded in audit report. | | AUDIT-0197-T | DONE | Revalidated 2026-01-06; open findings recorded in audit report. | | AUDIT-0197-A | TODO | Revalidated 2026-01-06; open findings pending approval. | diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Apple/Jobs.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Apple/Jobs.cs index 11d912e10..7721db388 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Apple/Jobs.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Apple/Jobs.cs @@ -8,9 +8,9 @@ namespace StellaOps.Concelier.Connector.Vndr.Apple; internal static class AppleJobKinds { - public const string Fetch = "source:vndr-apple:fetch"; - public const string Parse = "source:vndr-apple:parse"; - public const string Map = "source:vndr-apple:map"; + public const string Fetch = "source:apple:fetch"; + public const string Parse = "source:apple:parse"; + public const string Map = "source:apple:map"; } internal sealed class AppleFetchJob : IJob diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Chromium/ChromiumDependencyInjectionRoutine.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Chromium/ChromiumDependencyInjectionRoutine.cs new file mode 100644 index 000000000..10d8e0e14 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Chromium/ChromiumDependencyInjectionRoutine.cs @@ -0,0 +1,57 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.DependencyInjection; +using System; + +namespace StellaOps.Concelier.Connector.Vndr.Chromium; + +public sealed class ChromiumDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "concelier:sources:chromium"; + private const string LegacyConfigurationSection = "concelier:sources:vndr-chromium"; + private const string LegacyVendorConfigurationSection = "concelier:sources:vndr:chromium"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddChromiumConnector(options => + { + configuration.GetSection(LegacyVendorConfigurationSection).Bind(options); + configuration.GetSection(LegacyConfigurationSection).Bind(options); + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, ChromiumJobKinds.Fetch, typeof(ChromiumFetchJob)); + EnsureJob(options, ChromiumJobKinds.Parse, typeof(ChromiumParseJob)); + EnsureJob(options, ChromiumJobKinds.Map, typeof(ChromiumMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Chromium/Jobs.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Chromium/Jobs.cs new file mode 100644 index 000000000..0a6872aa5 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Chromium/Jobs.cs @@ -0,0 +1,46 @@ +using StellaOps.Concelier.Core.Jobs; +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Concelier.Connector.Vndr.Chromium; + +internal static class ChromiumJobKinds +{ + public const string Fetch = "source:chromium:fetch"; + public const string Parse = "source:chromium:parse"; + public const string Map = "source:chromium:map"; +} + +internal sealed class ChromiumFetchJob : IJob +{ + private readonly ChromiumConnector _connector; + + public ChromiumFetchJob(ChromiumConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class ChromiumParseJob : IJob +{ + private readonly ChromiumConnector _connector; + + public ChromiumParseJob(ChromiumConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class ChromiumMapJob : IJob +{ + private readonly ChromiumConnector _connector; + + public ChromiumMapJob(ChromiumConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Chromium/TASKS.md b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Chromium/TASKS.md index 10be3644d..447cf2c26 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Chromium/TASKS.md +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Chromium/TASKS.md @@ -5,6 +5,7 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | Task ID | Status | Notes | | --- | --- | --- | +| CONN-ALIGN-007 | DONE | 2026-04-22: Promoted Chromium into the canonical `chromium` catalog/runtime path with built-in host jobs, alias normalization, and updated operator runbooks. | | AUDIT-0201-M | DONE | Revalidated 2026-01-06. | | AUDIT-0201-T | DONE | Revalidated 2026-01-06. | | AUDIT-0201-A | TODO | Revalidated 2026-01-06 (open findings). | diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/CiscoConnector.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/CiscoConnector.cs index 6dd2c27c2..208074d15 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/CiscoConnector.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/CiscoConnector.cs @@ -47,7 +47,7 @@ public sealed class CiscoConnector : IFeedConnector private readonly CiscoDtoFactory _dtoFactory; private readonly ICryptoHash _hash; private readonly CiscoDiagnostics _diagnostics; - private readonly IOptions _options; + private readonly IOptionsMonitor _options; private readonly TimeProvider _timeProvider; private readonly ILogger _logger; @@ -61,7 +61,7 @@ public sealed class CiscoConnector : IFeedConnector CiscoDtoFactory dtoFactory, ICryptoHash hash, CiscoDiagnostics diagnostics, - IOptions options, + IOptionsMonitor options, TimeProvider? timeProvider, ILogger logger) { @@ -105,7 +105,7 @@ public sealed class CiscoConnector : IFeedConnector ArgumentNullException.ThrowIfNull(services); var now = _timeProvider.GetUtcNow(); - var options = _options.Value; + var options = GetOptions(); var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); var pendingDocuments = cursor.PendingDocuments.ToHashSet(); var pendingMappings = cursor.PendingMappings.ToHashSet(); @@ -584,7 +584,7 @@ public sealed class CiscoConnector : IFeedConnector private string BuildDocumentUri(string advisoryId) { - var baseUri = _options.Value.BaseUri; + var baseUri = GetOptions().BaseUri; var relative = $"advisories/{Uri.EscapeDataString(advisoryId)}"; return new Uri(baseUri, relative).ToString(); } @@ -601,6 +601,13 @@ public sealed class CiscoConnector : IFeedConnector await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); } + private CiscoOptions GetOptions() + { + var options = _options.CurrentValue; + options.Validate(); + return options; + } + private static Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken) { if (delay <= TimeSpan.Zero) diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/CiscoDependencyInjectionRoutine.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/CiscoDependencyInjectionRoutine.cs index 582b8a380..bb9e2484c 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/CiscoDependencyInjectionRoutine.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/CiscoDependencyInjectionRoutine.cs @@ -19,7 +19,6 @@ public sealed class CiscoDependencyInjectionRoutine : IDependencyInjectionRoutin services.AddCiscoConnector(options => { configuration.GetSection(ConfigurationSection).Bind(options); - options.Validate(); }); services.AddTransient(); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/CiscoServiceCollectionExtensions.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/CiscoServiceCollectionExtensions.cs index 59931c9a2..676e36aef 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/CiscoServiceCollectionExtensions.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/CiscoServiceCollectionExtensions.cs @@ -18,19 +18,18 @@ public static class CiscoServiceCollectionExtensions ArgumentNullException.ThrowIfNull(configure); services.AddOptions() - .Configure(configure) - .PostConfigure(static opts => opts.Validate()) - .ValidateOnStart(); + .Configure(configure); services.TryAddSingleton(_ => TimeProvider.System); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); services.AddTransient(); services.AddHttpClient(CiscoOptions.AuthHttpClientName) .ConfigureHttpClient((sp, client) => { - var options = sp.GetRequiredService>().Value; + var options = sp.GetRequiredService>().CurrentValue; client.Timeout = options.RequestTimeout; client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Concelier.Cisco/1.0"); client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); @@ -42,7 +41,7 @@ public static class CiscoServiceCollectionExtensions services.AddSourceHttpClient(CiscoOptions.HttpClientName, static (sp, clientOptions) => { - var options = sp.GetRequiredService>().Value; + var options = sp.GetRequiredService>().CurrentValue; clientOptions.Timeout = options.RequestTimeout; clientOptions.UserAgent = "StellaOps.Concelier.Cisco/1.0"; clientOptions.AllowedHosts.Clear(); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoAccessTokenProvider.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoAccessTokenProvider.cs index b402443f4..9561bbad3 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoAccessTokenProvider.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoAccessTokenProvider.cs @@ -8,7 +8,14 @@ using System.Text.Json.Serialization; namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal; -internal sealed class CiscoAccessTokenProvider : IDisposable +public interface ICiscoAccessTokenProvider +{ + Task GetTokenAsync(CancellationToken cancellationToken); + Task RefreshAsync(CancellationToken cancellationToken); + void Invalidate(); +} + +internal sealed class CiscoAccessTokenProvider : ICiscoAccessTokenProvider, IDisposable { private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) { diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoOAuthMessageHandler.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoOAuthMessageHandler.cs index 00a6d996a..17dc616a9 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoOAuthMessageHandler.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoOAuthMessageHandler.cs @@ -8,11 +8,11 @@ namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal; internal sealed class CiscoOAuthMessageHandler : DelegatingHandler { - private readonly CiscoAccessTokenProvider _tokenProvider; + private readonly ICiscoAccessTokenProvider _tokenProvider; private readonly ILogger _logger; public CiscoOAuthMessageHandler( - CiscoAccessTokenProvider tokenProvider, + ICiscoAccessTokenProvider tokenProvider, ILogger logger) { _tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider)); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/Jobs.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/Jobs.cs index 52adb52f1..aac789951 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/Jobs.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/Jobs.cs @@ -8,9 +8,9 @@ namespace StellaOps.Concelier.Connector.Vndr.Cisco; internal static class CiscoJobKinds { - public const string Fetch = "source:vndr-cisco:fetch"; - public const string Parse = "source:vndr-cisco:parse"; - public const string Map = "source:vndr-cisco:map"; + public const string Fetch = "source:cisco:fetch"; + public const string Parse = "source:cisco:parse"; + public const string Map = "source:cisco:map"; } internal sealed class CiscoFetchJob : IJob diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/AGENTS.md b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/AGENTS.md index 9eb7685f3..5ca73926c 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/AGENTS.md +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/AGENTS.md @@ -13,7 +13,7 @@ Implement the Microsoft Security Response Center (MSRC) connector to ingest Micr - `Source.Common`, `Storage.Postgres`, `Concelier.Models`, `Concelier.Testing`. ## Interfaces & Contracts -- Job kinds: `msrc:fetch`, `msrc:parse`, `msrc:map`. +- Job kinds: `source:microsoft:fetch`, `source:microsoft:parse`, `source:microsoft:map`. - Persist upstream metadata (e.g., `lastModified`, `releaseDate`). - Alias set should include MSRC ID, CVEs, and KB identifiers. diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcApiClient.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcApiClient.cs index b253f8572..8dcca8cdb 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcApiClient.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcApiClient.cs @@ -24,18 +24,18 @@ public sealed class MsrcApiClient private readonly IHttpClientFactory _httpClientFactory; private readonly IMsrcTokenProvider _tokenProvider; - private readonly MsrcOptions _options; + private readonly IOptionsMonitor _options; private readonly ILogger _logger; public MsrcApiClient( IHttpClientFactory httpClientFactory, IMsrcTokenProvider tokenProvider, - IOptions options, + IOptionsMonitor options, ILogger logger) { _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); _tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider)); - _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options = options ?? throw new ArgumentNullException(nameof(options)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -98,6 +98,7 @@ public sealed class MsrcApiClient private async Task CreateAuthenticatedClientAsync(CancellationToken cancellationToken) { + var options = GetOptions(); var token = await _tokenProvider.GetAccessTokenAsync(cancellationToken).ConfigureAwait(false); var client = _httpClientFactory.CreateClient(MsrcOptions.HttpClientName); client.DefaultRequestHeaders.Remove("Authorization"); @@ -105,23 +106,24 @@ public sealed class MsrcApiClient client.DefaultRequestHeaders.Remove("Accept"); client.DefaultRequestHeaders.Add("Accept", "application/json"); client.DefaultRequestHeaders.Remove("api-version"); - client.DefaultRequestHeaders.Add("api-version", _options.ApiVersion); + client.DefaultRequestHeaders.Add("api-version", options.ApiVersion); client.DefaultRequestHeaders.Remove("Accept-Language"); - client.DefaultRequestHeaders.Add("Accept-Language", _options.Locale); + client.DefaultRequestHeaders.Add("Accept-Language", options.Locale); return client; } private Uri BuildSummaryUri(DateTimeOffset fromInclusive, DateTimeOffset toExclusive) { + var options = GetOptions(); var builder = new StringBuilder(); - builder.Append(_options.BaseUri.ToString().TrimEnd('/')); + builder.Append(options.BaseUri.ToString().TrimEnd('/')); builder.Append("/vulnerabilities?"); - builder.Append("$top=").Append(_options.PageSize); + builder.Append("$top=").Append(options.PageSize); builder.Append("&lastModifiedStartDateTime=").Append(Uri.EscapeDataString(fromInclusive.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture))); builder.Append("&lastModifiedEndDateTime=").Append(Uri.EscapeDataString(toExclusive.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture))); builder.Append("&$orderby=lastModifiedDate"); - builder.Append("&locale=").Append(Uri.EscapeDataString(_options.Locale)); - builder.Append("&api-version=").Append(Uri.EscapeDataString(_options.ApiVersion)); + builder.Append("&locale=").Append(Uri.EscapeDataString(options.Locale)); + builder.Append("&api-version=").Append(Uri.EscapeDataString(options.ApiVersion)); return new Uri(builder.ToString(), UriKind.Absolute); } @@ -133,8 +135,16 @@ public sealed class MsrcApiClient throw new ArgumentException("Vulnerability identifier must be provided.", nameof(vulnerabilityId)); } - var baseUri = _options.BaseUri.ToString().TrimEnd('/'); - var path = $"{baseUri}/vulnerability/{Uri.EscapeDataString(vulnerabilityId)}?api-version={Uri.EscapeDataString(_options.ApiVersion)}&locale={Uri.EscapeDataString(_options.Locale)}"; + var options = GetOptions(); + var baseUri = options.BaseUri.ToString().TrimEnd('/'); + var path = $"{baseUri}/vulnerability/{Uri.EscapeDataString(vulnerabilityId)}?api-version={Uri.EscapeDataString(options.ApiVersion)}&locale={Uri.EscapeDataString(options.Locale)}"; return new Uri(path, UriKind.Absolute); } + + private MsrcOptions GetOptions() + { + var options = _options.CurrentValue; + options.Validate(); + return options; + } } diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcTokenProvider.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcTokenProvider.cs index fd1cddea7..26cee4a9f 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcTokenProvider.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcTokenProvider.cs @@ -15,12 +15,13 @@ namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal; public interface IMsrcTokenProvider { Task GetAccessTokenAsync(CancellationToken cancellationToken); + void Invalidate(); } public sealed class MsrcTokenProvider : IMsrcTokenProvider, IDisposable { private readonly IHttpClientFactory _httpClientFactory; - private readonly MsrcOptions _options; + private readonly IOptionsMonitor _options; private readonly TimeProvider _timeProvider; private readonly ILogger _logger; private readonly SemaphoreSlim _refreshLock = new(1, 1); @@ -29,13 +30,12 @@ public sealed class MsrcTokenProvider : IMsrcTokenProvider, IDisposable public MsrcTokenProvider( IHttpClientFactory httpClientFactory, - IOptions options, + IOptionsMonitor options, TimeProvider? timeProvider, ILogger logger) { _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); - _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); - _options.Validate(); + _options = options ?? throw new ArgumentNullException(nameof(options)); _timeProvider = timeProvider ?? TimeProvider.System; _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -58,15 +58,16 @@ public sealed class MsrcTokenProvider : IMsrcTokenProvider, IDisposable } _logger.LogInformation("Requesting new MSRC access token"); + var options = GetOptions(); var client = _httpClientFactory.CreateClient(MsrcOptions.TokenClientName); var request = new HttpRequestMessage(HttpMethod.Post, BuildTokenUri()) { Content = new FormUrlEncodedContent(new Dictionary { - ["client_id"] = _options.ClientId, - ["client_secret"] = _options.ClientSecret, + ["client_id"] = options.ClientId, + ["client_secret"] = options.ClientSecret, ["grant_type"] = "client_credentials", - ["scope"] = _options.Scope, + ["scope"] = options.Scope, }), }; @@ -86,11 +87,23 @@ public sealed class MsrcTokenProvider : IMsrcTokenProvider, IDisposable } } + public void Invalidate() + { + _currentToken = null; + } + private Uri BuildTokenUri() - => new($"https://login.microsoftonline.com/{_options.TenantId}/oauth2/v2.0/token"); + => new($"https://login.microsoftonline.com/{GetOptions().TenantId}/oauth2/v2.0/token"); public void Dispose() => _refreshLock.Dispose(); + private MsrcOptions GetOptions() + { + var options = _options.CurrentValue; + options.Validate(); + return options; + } + private sealed record AccessToken(string Token, DateTimeOffset ExpiresAt) { public bool IsExpired(DateTimeOffset now) => now >= ExpiresAt; diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/Jobs.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/Jobs.cs index b1085ac2f..443e7e560 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/Jobs.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/Jobs.cs @@ -8,7 +8,9 @@ namespace StellaOps.Concelier.Connector.Vndr.Msrc; internal static class MsrcJobKinds { - public const string Fetch = "source:vndr.msrc:fetch"; + public const string Fetch = "source:microsoft:fetch"; + public const string Parse = "source:microsoft:parse"; + public const string Map = "source:microsoft:map"; } internal sealed class MsrcFetchJob : IJob @@ -21,3 +23,25 @@ internal sealed class MsrcFetchJob : IJob public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) => _connector.FetchAsync(context.Services, cancellationToken); } + +internal sealed class MsrcParseJob : IJob +{ + private readonly MsrcConnector _connector; + + public MsrcParseJob(MsrcConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class MsrcMapJob : IJob +{ + private readonly MsrcConnector _connector; + + public MsrcMapJob(MsrcConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcConnector.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcConnector.cs index 99ac81ee0..936d6d9d5 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcConnector.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcConnector.cs @@ -39,7 +39,7 @@ public sealed class MsrcConnector : IFeedConnector private readonly IDtoStore _dtoStore; private readonly IAdvisoryStore _advisoryStore; private readonly ISourceStateRepository _stateRepository; - private readonly MsrcOptions _options; + private readonly IOptionsMonitor _options; private readonly ICryptoHash _hash; private readonly TimeProvider _timeProvider; private readonly ILogger _logger; @@ -55,7 +55,7 @@ public sealed class MsrcConnector : IFeedConnector IAdvisoryStore advisoryStore, ISourceStateRepository stateRepository, ICryptoHash hash, - IOptions options, + IOptionsMonitor options, TimeProvider? timeProvider, MsrcDiagnostics diagnostics, ILogger logger) @@ -69,8 +69,7 @@ public sealed class MsrcConnector : IFeedConnector _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); _hash = hash ?? throw new ArgumentNullException(nameof(hash)); - _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); - _options.Validate(); + _options = options ?? throw new ArgumentNullException(nameof(options)); _timeProvider = timeProvider ?? TimeProvider.System; _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -101,10 +100,11 @@ public sealed class MsrcConnector : IFeedConnector { ArgumentNullException.ThrowIfNull(services); + var options = GetOptions(); var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); var now = _timeProvider.GetUtcNow(); - var from = cursor.LastModifiedCursor ?? _options.InitialLastModified ?? now.AddDays(-30); - from = from.Add(-_options.CursorOverlap); + var from = cursor.LastModifiedCursor ?? options.InitialLastModified ?? now.AddDays(-30); + from = from.Add(-options.CursorOverlap); var to = now; _diagnostics.SummaryFetchAttempt(); @@ -120,7 +120,7 @@ public sealed class MsrcConnector : IFeedConnector { _diagnostics.SummaryFetchFailure("exception"); _logger.LogError(ex, "MSRC summary fetch failed"); - await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); + await _stateRepository.MarkFailureAsync(SourceName, now, options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); throw; } @@ -138,7 +138,7 @@ public sealed class MsrcConnector : IFeedConnector foreach (var summary in summaries.OrderBy(static s => s.LastModifiedDate ?? DateTimeOffset.MinValue)) { cancellationToken.ThrowIfCancellationRequested(); - if (processed >= _options.MaxAdvisoriesPerFetch) + if (processed >= options.MaxAdvisoriesPerFetch) { break; } @@ -203,14 +203,14 @@ public sealed class MsrcConnector : IFeedConnector _diagnostics.DetailFetchSuccess(); processed++; - if (_options.DownloadCvrf) + if (options.DownloadCvrf) { await FetchCvrfAsync(summary, now, cancellationToken).ConfigureAwait(false); } - if (_options.RequestDelay > TimeSpan.Zero) + if (options.RequestDelay > TimeSpan.Zero) { - await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + await Task.Delay(options.RequestDelay, cancellationToken).ConfigureAwait(false); } } catch (Exception ex) @@ -218,7 +218,7 @@ public sealed class MsrcConnector : IFeedConnector _diagnostics.DetailFetchFailure("exception"); failures++; _logger.LogError(ex, "MSRC detail fetch failed for {VulnerabilityId}", vulnerabilityId); - await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); + await _stateRepository.MarkFailureAsync(SourceName, now, options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); throw; } } @@ -476,4 +476,11 @@ public sealed class MsrcConnector : IFeedConnector var completedAt = _timeProvider.GetUtcNow(); return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken); } + + private MsrcOptions GetOptions() + { + var options = _options.CurrentValue; + options.Validate(); + return options; + } } diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcDependencyInjectionRoutine.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcDependencyInjectionRoutine.cs index da3b5f99d..0d3a56f88 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcDependencyInjectionRoutine.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcDependencyInjectionRoutine.cs @@ -10,7 +10,8 @@ namespace StellaOps.Concelier.Connector.Vndr.Msrc; public sealed class MsrcDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "concelier:sources:vndr:msrc"; + private const string ConfigurationSection = "concelier:sources:microsoft"; + private const string LegacyConfigurationSection = "concelier:sources:vndr:msrc"; public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) { @@ -19,15 +20,19 @@ public sealed class MsrcDependencyInjectionRoutine : IDependencyInjectionRoutine services.AddMsrcConnector(options => { + configuration.GetSection(LegacyConfigurationSection).Bind(options); configuration.GetSection(ConfigurationSection).Bind(options); - options.Validate(); }); services.AddTransient(); + services.AddTransient(); + services.AddTransient(); services.PostConfigure(options => { EnsureJob(options, MsrcJobKinds.Fetch, typeof(MsrcFetchJob)); + EnsureJob(options, MsrcJobKinds.Parse, typeof(MsrcParseJob)); + EnsureJob(options, MsrcJobKinds.Map, typeof(MsrcMapJob)); }); return services; diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcServiceCollectionExtensions.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcServiceCollectionExtensions.cs index 6b9027898..b458675a6 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcServiceCollectionExtensions.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcServiceCollectionExtensions.cs @@ -18,12 +18,11 @@ public static class MsrcServiceCollectionExtensions ArgumentNullException.ThrowIfNull(configure); services.AddOptions() - .Configure(configure) - .PostConfigure(static options => options.Validate()); + .Configure(configure); services.AddSourceHttpClient(MsrcOptions.HttpClientName, static (provider, clientOptions) => { - var options = provider.GetRequiredService>().Value; + var options = provider.GetRequiredService>().CurrentValue; clientOptions.Timeout = TimeSpan.FromSeconds(30); clientOptions.AllowedHosts.Clear(); clientOptions.AllowedHosts.Add(options.BaseUri.Host); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/TASKS.md b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/TASKS.md index 8367a7440..eefcdd0c6 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/TASKS.md +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Msrc/TASKS.md @@ -5,6 +5,7 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | Task ID | Status | Notes | | --- | --- | --- | +| CONN-ALIGN-005 | DONE | 2026-04-22 runtime-alignment sprint exposed host fetch/parse/map registration for `source:microsoft:*` and accepted canonical `concelier:sources:microsoft` configuration. | | AUDIT-0205-M | DONE | Revalidated 2026-01-06. | | AUDIT-0205-T | DONE | Revalidated 2026-01-06. | | AUDIT-0205-A | TODO | Revalidated 2026-01-06 (open findings). | diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/Configuration/OracleOptions.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/Configuration/OracleOptions.cs index ebabd20e9..74655f27f 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/Configuration/OracleOptions.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/Configuration/OracleOptions.cs @@ -10,7 +10,10 @@ public sealed class OracleOptions public List AdvisoryUris { get; set; } = new(); - public List CalendarUris { get; set; } = new(); + public List CalendarUris { get; set; } = + [ + new Uri("https://www.oracle.com/security-alerts/", UriKind.Absolute), + ]; public TimeSpan RequestDelay { get; set; } = TimeSpan.FromSeconds(1); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleCalendarFetcher.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleCalendarFetcher.cs index e0c6f59a2..0ddd5127a 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleCalendarFetcher.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleCalendarFetcher.cs @@ -15,24 +15,26 @@ namespace StellaOps.Concelier.Connector.Vndr.Oracle.Internal; public sealed class OracleCalendarFetcher { private static readonly Regex AnchorRegex = new("]+href=\"(?[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex AdvisoryPathRegex = new(@"^/security-alerts/[^/?#]+\.html$", RegexOptions.IgnoreCase | RegexOptions.Compiled); private readonly IHttpClientFactory _httpClientFactory; - private readonly OracleOptions _options; + private readonly IOptionsMonitor _options; private readonly ILogger _logger; public OracleCalendarFetcher( IHttpClientFactory httpClientFactory, - IOptions options, + IOptionsMonitor options, ILogger logger) { _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); - _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options = options ?? throw new ArgumentNullException(nameof(options)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public async Task> GetAdvisoryUrisAsync(CancellationToken cancellationToken) { - if (_options.CalendarUris.Count == 0) + var options = GetOptions(); + if (options.CalendarUris.Count == 0) { return Array.Empty(); } @@ -40,7 +42,7 @@ public sealed class OracleCalendarFetcher var discovered = new HashSet(StringComparer.OrdinalIgnoreCase); var client = _httpClientFactory.CreateClient(OracleOptions.HttpClientName); - foreach (var calendarUri in _options.CalendarUris) + foreach (var calendarUri in options.CalendarUris) { try { @@ -62,6 +64,13 @@ public sealed class OracleCalendarFetcher .ToArray(); } + private OracleOptions GetOptions() + { + var options = _options.CurrentValue; + options.Validate(); + return options; + } + private static IEnumerable ExtractLinks(Uri baseUri, string html) { if (string.IsNullOrWhiteSpace(html)) @@ -87,7 +96,24 @@ public sealed class OracleCalendarFetcher continue; } + if (!IsAdvisoryUri(uri)) + { + continue; + } + yield return uri; } } + + private static bool IsAdvisoryUri(Uri uri) + { + if (!uri.IsAbsoluteUri) + { + return false; + } + + return (string.Equals(uri.Host, "www.oracle.com", StringComparison.OrdinalIgnoreCase) || + string.Equals(uri.Host, "oracle.com", StringComparison.OrdinalIgnoreCase)) && + AdvisoryPathRegex.IsMatch(uri.AbsolutePath); + } } diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/Jobs.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/Jobs.cs index 9e6746649..8bf072f53 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/Jobs.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/Jobs.cs @@ -8,9 +8,9 @@ namespace StellaOps.Concelier.Connector.Vndr.Oracle; internal static class OracleJobKinds { - public const string Fetch = "source:vndr-oracle:fetch"; - public const string Parse = "source:vndr-oracle:parse"; - public const string Map = "source:vndr-oracle:map"; + public const string Fetch = "source:oracle:fetch"; + public const string Parse = "source:oracle:parse"; + public const string Map = "source:oracle:map"; } internal sealed class OracleFetchJob : IJob diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/OracleConnector.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/OracleConnector.cs index b48dbda94..482c5c956 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/OracleConnector.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/OracleConnector.cs @@ -39,7 +39,7 @@ public sealed class OracleConnector : IFeedConnector private readonly ISourceStateRepository _stateRepository; private readonly OracleCalendarFetcher _calendarFetcher; private readonly ICryptoHash _hash; - private readonly OracleOptions _options; + private readonly IOptionsMonitor _options; private readonly TimeProvider _timeProvider; private readonly ILogger _logger; @@ -53,7 +53,7 @@ public sealed class OracleConnector : IFeedConnector ISourceStateRepository stateRepository, OracleCalendarFetcher calendarFetcher, ICryptoHash hash, - IOptions options, + IOptionsMonitor options, TimeProvider? timeProvider, ILogger logger) { @@ -66,8 +66,7 @@ public sealed class OracleConnector : IFeedConnector _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); _calendarFetcher = calendarFetcher ?? throw new ArgumentNullException(nameof(calendarFetcher)); _hash = hash ?? throw new ArgumentNullException(nameof(hash)); - _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); - _options.Validate(); + _options = options ?? throw new ArgumentNullException(nameof(options)); _timeProvider = timeProvider ?? TimeProvider.System; _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -86,6 +85,7 @@ public sealed class OracleConnector : IFeedConnector public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) { + var options = GetOptions(); var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); var pendingDocuments = cursor.PendingDocuments.ToList(); var pendingMappings = cursor.PendingMappings.ToList(); @@ -146,9 +146,9 @@ public sealed class OracleConnector : IFeedConnector pendingDocuments.Add(result.Document.Id); } - if (_options.RequestDelay > TimeSpan.Zero) + if (options.RequestDelay > TimeSpan.Zero) { - await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + await Task.Delay(options.RequestDelay, cancellationToken).ConfigureAwait(false); } } catch (Exception ex) @@ -342,7 +342,7 @@ public sealed class OracleConnector : IFeedConnector { var uris = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (var uri in _options.AdvisoryUris) + foreach (var uri in GetOptions().AdvisoryUris) { if (uri is not null) { @@ -378,4 +378,11 @@ public sealed class OracleConnector : IFeedConnector return slug.Replace('.', '-'); } + + private OracleOptions GetOptions() + { + var options = _options.CurrentValue; + options.Validate(); + return options; + } } diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/OracleDependencyInjectionRoutine.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/OracleDependencyInjectionRoutine.cs index d713ff211..24f5e5f90 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/OracleDependencyInjectionRoutine.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/OracleDependencyInjectionRoutine.cs @@ -20,7 +20,6 @@ public sealed class OracleDependencyInjectionRoutine : IDependencyInjectionRouti services.AddOracleConnector(options => { configuration.GetSection(ConfigurationSection).Bind(options); - options.Validate(); }); services.AddTransient(); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/OracleServiceCollectionExtensions.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/OracleServiceCollectionExtensions.cs index db7483f0a..ec7d6eb1a 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/OracleServiceCollectionExtensions.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Oracle/OracleServiceCollectionExtensions.cs @@ -17,12 +17,11 @@ public static class OracleServiceCollectionExtensions ArgumentNullException.ThrowIfNull(configure); services.AddOptions() - .Configure(configure) - .PostConfigure(static opts => opts.Validate()); + .Configure(configure); services.AddSourceHttpClient(OracleOptions.HttpClientName, static (sp, clientOptions) => { - var options = sp.GetRequiredService>().Value; + var options = sp.GetRequiredService>().CurrentValue; clientOptions.Timeout = TimeSpan.FromSeconds(30); clientOptions.UserAgent = "StellaOps.Concelier.Oracle/1.0"; clientOptions.AllowedHosts.Clear(); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Vmware/Configuration/VmwareOptions.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Vmware/Configuration/VmwareOptions.cs index 5cc23d678..49ec7130c 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Vmware/Configuration/VmwareOptions.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Vmware/Configuration/VmwareOptions.cs @@ -6,7 +6,7 @@ public sealed class VmwareOptions { public const string HttpClientName = "source.vmware"; - public Uri IndexUri { get; set; } = new("https://example.invalid/vmsa/index.json", UriKind.Absolute); + public Uri IndexUri { get; set; } = new("https://support.broadcom.com/web/ecx/security-advisory?segment=VC", UriKind.Absolute); public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Vmware/Internal/VmwareHtmlParser.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Vmware/Internal/VmwareHtmlParser.cs new file mode 100644 index 000000000..8963bad8b --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Vmware/Internal/VmwareHtmlParser.cs @@ -0,0 +1,430 @@ +using System.Globalization; +using System.Net; +using System.Text.RegularExpressions; + +namespace StellaOps.Concelier.Connector.Vndr.Vmware.Internal; + +internal static partial class VmwareHtmlParser +{ + private static readonly string[] ProductSectionStops = + { + "## ", + "Notification Id", + "Last Updated", + "Initial Publication Date", + "Status", + "Severity", + "CVSS Base Score", + "WorkAround", + "Affected CVE", + "Advisory ID:", + }; + + public static VmwareDetailDto? Parse(string html, Uri detailUri) + { + if (string.IsNullOrWhiteSpace(html)) + { + return null; + } + + var lines = ConvertHtmlToLines(html); + if (lines.Count == 0) + { + return null; + } + + var title = lines.FirstOrDefault(static line => AdvisoryIdRegex().IsMatch(line))?.Trim(); + title ??= detailUri.ToString(); + + var advisoryId = NormalizeAdvisoryId( + ExtractLabeledValue(lines, "Advisory ID:") + ?? AdvisoryIdRegex().Match(title).Value + ?? AdvisoryIdRegex().Match(string.Join(' ', lines.Take(16))).Value); + + if (string.IsNullOrWhiteSpace(advisoryId)) + { + return null; + } + + var summary = ExtractLabeledValue(lines, "Synopsis:") + ?? ExtractSectionText(lines, "## 2. Introduction") + ?? title; + + var published = ParseDate( + ExtractLabeledValue(lines, "Initial Publication Date") + ?? ExtractLabeledValue(lines, "Issue date:")); + + var modified = ParseDate( + ExtractLabeledValue(lines, "Last Updated") + ?? ExtractLabeledValue(lines, "Updated on:")); + + var cves = ExtractCves(lines); + var affected = ExtractAffectedProducts(lines); + var references = ExtractReferences(html, detailUri); + + return new VmwareDetailDto + { + AdvisoryId = advisoryId, + Title = title, + Summary = string.Equals(summary, title, StringComparison.Ordinal) ? null : summary, + Published = published, + Modified = modified, + CveIds = cves, + Affected = affected, + References = references, + }; + } + + private static IReadOnlyList ConvertHtmlToLines(string html) + { + var withoutScripts = ScriptRegex().Replace(html, string.Empty); + var withoutStyles = StyleRegex().Replace(withoutScripts, string.Empty); + var withoutComments = CommentRegex().Replace(withoutStyles, string.Empty); + var withListMarkers = ListItemRegex().Replace(withoutComments, "- "); + var withLineBreaks = BlockBreakRegex().Replace(withListMarkers, "\n"); + var withTableSeparators = TableCellBreakRegex().Replace(withLineBreaks, "\t"); + var withoutTags = TagRegex().Replace(withTableSeparators, " "); + var decoded = WebUtility.HtmlDecode(withoutTags) + .Replace('\u00A0', ' ') + .Replace("\r\n", "\n", StringComparison.Ordinal) + .Replace('\r', '\n'); + + return decoded + .Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Select(static line => WhitespaceRegex().Replace(line, " ").Trim()) + .Where(static line => !string.IsNullOrWhiteSpace(line)) + .ToArray(); + } + + private static string? ExtractLabeledValue(IReadOnlyList lines, string label) + { + for (var i = 0; i < lines.Count; i++) + { + var line = lines[i]; + if (!line.StartsWith(label, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var inlineValue = line[label.Length..].Trim(); + if (!string.IsNullOrWhiteSpace(inlineValue)) + { + return inlineValue; + } + + for (var j = i + 1; j < lines.Count; j++) + { + if (!string.IsNullOrWhiteSpace(lines[j])) + { + return lines[j]; + } + } + } + + return null; + } + + private static string? ExtractSectionText(IReadOnlyList lines, string heading) + { + var start = -1; + for (var i = 0; i < lines.Count; i++) + { + if (string.Equals(lines[i], heading, StringComparison.OrdinalIgnoreCase)) + { + start = i + 1; + break; + } + } + + if (start < 0) + { + return null; + } + + var buffer = new List(); + for (var i = start; i < lines.Count; i++) + { + var line = lines[i]; + if (line.StartsWith("## ", StringComparison.Ordinal)) + { + break; + } + + if (!string.IsNullOrWhiteSpace(line)) + { + buffer.Add(line.TrimStart('-', ' ')); + } + } + + return buffer.Count == 0 ? null : string.Join(' ', buffer); + } + + private static DateTimeOffset? ParseDate(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var cleaned = value.Trim(); + var revisionMarker = cleaned.IndexOf(" (", StringComparison.Ordinal); + if (revisionMarker > 0) + { + cleaned = cleaned[..revisionMarker].Trim(); + } + + if (DateTimeOffset.TryParse( + cleaned, + CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, + out var parsed)) + { + return parsed.ToUniversalTime(); + } + + if (DateOnly.TryParse(cleaned, CultureInfo.InvariantCulture, DateTimeStyles.None, out var dateOnly)) + { + return new DateTimeOffset(dateOnly.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero); + } + + return null; + } + + private static IReadOnlyList ExtractCves(IReadOnlyList lines) + { + var cves = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var line in lines) + { + foreach (Match match in CveRegex().Matches(line)) + { + if (match.Success) + { + cves.Add(match.Value.ToUpperInvariant()); + } + } + } + + return cves.Count == 0 + ? Array.Empty() + : cves.OrderBy(static value => value, StringComparer.Ordinal).ToArray(); + } + + private static IReadOnlyList ExtractAffectedProducts(IReadOnlyList lines) + { + var products = new HashSet(StringComparer.OrdinalIgnoreCase); + CaptureProductsFromSection(lines, "## 1. Impacted Products", products); + if (products.Count == 0) + { + CaptureProductsFromSection(lines, "List of Products", products); + } + + return products.Count == 0 + ? Array.Empty() + : products + .OrderBy(static value => value, StringComparer.OrdinalIgnoreCase) + .Select(static product => new VmwareAffectedProductDto { Product = product }) + .ToArray(); + } + + private static void CaptureProductsFromSection(IReadOnlyList lines, string heading, HashSet products) + { + var start = -1; + for (var i = 0; i < lines.Count; i++) + { + if (string.Equals(lines[i], heading, StringComparison.OrdinalIgnoreCase)) + { + start = i + 1; + break; + } + } + + if (start < 0) + { + return; + } + + for (var i = start; i < lines.Count; i++) + { + var line = lines[i]; + if (ProductSectionStops.Any(stop => line.StartsWith(stop, StringComparison.OrdinalIgnoreCase))) + { + break; + } + + var cleaned = line.TrimStart('-', ' ').Trim(); + if (string.IsNullOrWhiteSpace(cleaned) + || cleaned.EndsWith("Products", StringComparison.OrdinalIgnoreCase) + || cleaned.EndsWith("Product", StringComparison.OrdinalIgnoreCase) + || cleaned.Contains("more products", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + products.Add(cleaned); + } + } + + private static IReadOnlyList ExtractReferences(string html, Uri detailUri) + { + var references = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [detailUri.ToString()] = new VmwareReferenceDto + { + Type = "advisory", + Url = detailUri.ToString(), + }, + }; + + foreach (Match match in HrefRegex().Matches(html)) + { + if (!match.Success) + { + continue; + } + + var raw = WebUtility.HtmlDecode(match.Groups["url"].Value.Trim()); + if (!Uri.TryCreate(detailUri, raw, out var uri) || !uri.IsAbsoluteUri) + { + continue; + } + + if (!string.Equals(uri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) + && !string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (!ShouldIncludeReference(uri, detailUri)) + { + continue; + } + + references[uri.ToString()] = new VmwareReferenceDto + { + Type = InferReferenceType(uri, detailUri), + Url = uri.ToString(), + }; + } + + return references.Values + .OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static bool ShouldIncludeReference(Uri uri, Uri detailUri) + { + if (Uri.Compare(uri, detailUri, UriComponents.AbsoluteUri, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) == 0) + { + return true; + } + + var path = uri.AbsolutePath ?? string.Empty; + if (path.EndsWith(".css", StringComparison.OrdinalIgnoreCase) + || path.EndsWith(".js", StringComparison.OrdinalIgnoreCase) + || path.EndsWith(".png", StringComparison.OrdinalIgnoreCase) + || path.EndsWith(".jpg", StringComparison.OrdinalIgnoreCase) + || path.EndsWith(".jpeg", StringComparison.OrdinalIgnoreCase) + || path.EndsWith(".svg", StringComparison.OrdinalIgnoreCase) + || path.EndsWith(".ico", StringComparison.OrdinalIgnoreCase) + || path.EndsWith(".woff", StringComparison.OrdinalIgnoreCase) + || path.EndsWith(".woff2", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + if (uri.Host.EndsWith("cve.org", StringComparison.OrdinalIgnoreCase) + || uri.Host.EndsWith("first.org", StringComparison.OrdinalIgnoreCase) + || uri.Host.EndsWith("knowledge.broadcom.com", StringComparison.OrdinalIgnoreCase) + || uri.Host.EndsWith("techdocs.broadcom.com", StringComparison.OrdinalIgnoreCase) + || uri.Host.EndsWith("blogs.vmware.com", StringComparison.OrdinalIgnoreCase) + || string.Equals(uri.Host, "x.com", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + if (!uri.Host.EndsWith("broadcom.com", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + return path.Contains("/support-content-notification/", StringComparison.OrdinalIgnoreCase) + || path.Contains("/group/ecx/productfiles", StringComparison.OrdinalIgnoreCase) + || path.Contains("/support/vmware-", StringComparison.OrdinalIgnoreCase) + || path.Contains("/support/vmware", StringComparison.OrdinalIgnoreCase) + || path.Contains("/security-response", StringComparison.OrdinalIgnoreCase) + || path.Contains("/support-content-notification", StringComparison.OrdinalIgnoreCase); + } + + private static string? InferReferenceType(Uri uri, Uri detailUri) + { + if (Uri.Compare(uri, detailUri, UriComponents.AbsoluteUri, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) == 0) + { + return "advisory"; + } + + if (uri.Host.EndsWith("knowledge.broadcom.com", StringComparison.OrdinalIgnoreCase)) + { + return "kb"; + } + + if (uri.Host.EndsWith("techdocs.broadcom.com", StringComparison.OrdinalIgnoreCase) + || (uri.Host.EndsWith("broadcom.com", StringComparison.OrdinalIgnoreCase) + && uri.AbsolutePath.Contains("/group/ecx/productfiles", StringComparison.OrdinalIgnoreCase))) + { + return "patch"; + } + + return "advisory"; + } + + private static string NormalizeAdvisoryId(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return string.Empty; + } + + var match = AdvisoryIdRegex().Match(value); + if (!match.Success) + { + return string.Empty; + } + + var normalized = match.Value.ToUpperInvariant(); + var revisionSeparator = normalized.IndexOf('.', StringComparison.Ordinal); + return revisionSeparator >= 0 ? normalized[..revisionSeparator] : normalized; + } + + [GeneratedRegex("", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex ScriptRegex(); + + [GeneratedRegex("", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex StyleRegex(); + + [GeneratedRegex("", RegexOptions.Singleline | RegexOptions.CultureInvariant)] + private static partial Regex CommentRegex(); + + [GeneratedRegex("<\\s*li\\b[^>]*>", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex ListItemRegex(); + + [GeneratedRegex("<\\s*(?:br|/p|/div|/li|/tr|/table|/section|/article|/ul|/ol|/h[1-6])\\b[^>]*>", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex BlockBreakRegex(); + + [GeneratedRegex("<\\s*/t[dh]\\s*>", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex TableCellBreakRegex(); + + [GeneratedRegex("<[^>]+>", RegexOptions.CultureInvariant)] + private static partial Regex TagRegex(); + + [GeneratedRegex("\\s+", RegexOptions.CultureInvariant)] + private static partial Regex WhitespaceRegex(); + + [GeneratedRegex("CVE-\\d{4}-\\d{3,7}", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex CveRegex(); + + [GeneratedRegex("VMSA-\\d{4}-\\d{4}(?:\\.\\d+)?", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex AdvisoryIdRegex(); + + [GeneratedRegex("href\\s*=\\s*(?:\"(?[^\"]+)\"|'(?[^']+)'|(?[^\\s>]+))", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex HrefRegex(); +} diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Vmware/TASKS.md b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Vmware/TASKS.md index 3e9608987..6ad8d4c6c 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Vmware/TASKS.md +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Vmware/TASKS.md @@ -5,6 +5,7 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | Task ID | Status | Notes | | --- | --- | --- | +| CONN-ALIGN-003 | DOING | 2026-04-21 runtime validation: replacing the dead VMware default index with the live Broadcom advisory portal flow and adding HTML detail parsing for public VMSA pages. | | AUDIT-0209-M | DONE | Revalidated 2026-01-06. | | AUDIT-0209-T | DONE | Revalidated 2026-01-06. | | AUDIT-0209-A | TODO | Revalidated 2026-01-06 (open findings). | diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareConnector.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareConnector.cs index 754bfec33..fdade22c5 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareConnector.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareConnector.cs @@ -21,6 +21,7 @@ using System.Linq; using System.Net.Http; using System.Text; using System.Text.Json; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; @@ -28,6 +29,9 @@ namespace StellaOps.Concelier.Connector.Vndr.Vmware; public sealed class VmwareConnector : IFeedConnector { + private const int BroadcomPageSize = 100; + private const string BroadcomPortalPath = "/web/ecx/security-advisory"; + private const string BroadcomPortalApiPath = "/web/ecx/security-advisory/-/securityadvisory/getSecurityAdvisoryList"; private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) { PropertyNameCaseInsensitive = true, @@ -184,7 +188,7 @@ public sealed class VmwareConnector : IFeedConnector Metadata = metadata, ETag = existing?.Etag, LastModified = existing?.LastModified, - AcceptHeaders = new[] { "application/json" }, + AcceptHeaders = new[] { "text/html", "application/json" }, }, cancellationToken).ConfigureAwait(false); } @@ -338,7 +342,7 @@ public sealed class VmwareConnector : IFeedConnector VmwareDetailDto? detail; try { - detail = JsonSerializer.Deserialize(bytes, SerializerOptions); + detail = ParseDetailPayload(bytes, document.Uri); } catch (Exception ex) { @@ -448,6 +452,11 @@ public sealed class VmwareConnector : IFeedConnector private async Task> FetchIndexAsync(CancellationToken cancellationToken) { var client = _httpClientFactory.CreateClient(VmwareOptions.HttpClientName); + if (IsBroadcomPortalIndex(_options.IndexUri)) + { + return await FetchBroadcomPortalIndexAsync(client, cancellationToken).ConfigureAwait(false); + } + using var response = await client.GetAsync(_options.IndexUri, cancellationToken).ConfigureAwait(false); response.EnsureSuccessStatusCode(); @@ -456,6 +465,229 @@ public sealed class VmwareConnector : IFeedConnector return items ?? Array.Empty(); } + private static bool IsBroadcomPortalIndex(Uri indexUri) + => indexUri.AbsolutePath.Contains(BroadcomPortalPath, StringComparison.OrdinalIgnoreCase); + + private async Task> FetchBroadcomPortalIndexAsync(HttpClient client, CancellationToken cancellationToken) + { + using var landingResponse = await client.GetAsync(_options.IndexUri, cancellationToken).ConfigureAwait(false); + landingResponse.EnsureSuccessStatusCode(); + + var landingHtml = await landingResponse.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var csrfToken = ExtractBroadcomAuthToken(landingHtml); + if (string.IsNullOrWhiteSpace(csrfToken)) + { + throw new InvalidOperationException("Broadcom VMware advisory portal did not expose a CSRF token."); + } + + var segment = GetBroadcomSegment(_options.IndexUri); + var endpoint = new Uri(_options.IndexUri, BroadcomPortalApiPath); + var items = new Dictionary(StringComparer.OrdinalIgnoreCase); + + for (var pageNumber = 0; ; pageNumber++) + { + var payload = JsonSerializer.Serialize(new + { + pageNumber, + pageSize = BroadcomPageSize, + searchVal = string.Empty, + segment, + sortInfo = new + { + column = string.Empty, + order = string.Empty, + }, + }); + + using var request = new HttpRequestMessage(HttpMethod.Post, endpoint) + { + Content = new StringContent(payload, Encoding.UTF8, "application/json"), + }; + request.Headers.Referrer = _options.IndexUri; + request.Headers.TryAddWithoutValidation("X-CSRF-Token", csrfToken); + request.Headers.Accept.ParseAdd("application/json"); + + using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); + + if (!document.RootElement.TryGetProperty("data", out var data) + || !data.TryGetProperty("list", out var list) + || list.ValueKind != JsonValueKind.Array) + { + break; + } + + if (list.GetArrayLength() == 0) + { + break; + } + + foreach (var item in list.EnumerateArray()) + { + var detailUrl = GetString(item, "notificationUrl"); + if (string.IsNullOrWhiteSpace(detailUrl) + || !Uri.TryCreate(_options.IndexUri, detailUrl, out var detailUri) + || !detailUri.IsAbsoluteUri) + { + continue; + } + + var advisoryId = NormalizeVmwareAdvisoryId( + GetString(item, "title") + ?? GetString(item, "notificationId") + ?? GetString(item, "documentId")); + + if (string.IsNullOrWhiteSpace(advisoryId)) + { + continue; + } + + var modified = ParseBroadcomTimestamp(GetString(item, "updated")) + ?? ParseBroadcomTimestamp(GetString(item, "published")); + + items[advisoryId] = new VmwareIndexItem + { + Id = advisoryId, + DetailUrl = detailUri.ToString(), + Modified = modified, + }; + } + } + + return items.Count == 0 + ? Array.Empty() + : items.Values + .OrderBy(static item => item.Modified ?? DateTimeOffset.MinValue) + .ThenBy(static item => item.Id, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static VmwareDetailDto? ParseDetailPayload(byte[] bytes, string documentUri) + { + if (bytes.Length == 0) + { + return null; + } + + var firstNonWhitespace = 0; + while (firstNonWhitespace < bytes.Length && char.IsWhiteSpace((char)bytes[firstNonWhitespace])) + { + firstNonWhitespace++; + } + + if (firstNonWhitespace < bytes.Length && bytes[firstNonWhitespace] == (byte)'{') + { + return JsonSerializer.Deserialize(bytes, SerializerOptions); + } + + if (Uri.TryCreate(documentUri, UriKind.Absolute, out var detailUri)) + { + var html = Encoding.UTF8.GetString(bytes); + return VmwareHtmlParser.Parse(html, detailUri); + } + + return null; + } + + private static string? GetString(JsonElement element, string propertyName) + { + if (!element.TryGetProperty(propertyName, out var property)) + { + return null; + } + + return property.ValueKind switch + { + JsonValueKind.String => property.GetString(), + JsonValueKind.Number => property.ToString(), + _ => null, + }; + } + + private static string GetBroadcomSegment(Uri indexUri) + { + if (string.IsNullOrWhiteSpace(indexUri.Query)) + { + return "VC"; + } + + foreach (var part in indexUri.Query.TrimStart('?').Split('&', StringSplitOptions.RemoveEmptyEntries)) + { + var pieces = part.Split('=', 2); + if (pieces.Length != 2 || !string.Equals(pieces[0], "segment", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var value = Uri.UnescapeDataString(pieces[1]); + if (!string.IsNullOrWhiteSpace(value)) + { + return value; + } + } + + return "VC"; + } + + private static string? ExtractBroadcomAuthToken(string html) + { + var match = Regex.Match( + html, + "Liferay\\.authToken\\s*=\\s*['\\\"](?[^'\\\"]+)['\\\"]", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + + return match.Success ? match.Groups["token"].Value.Trim() : null; + } + + private static DateTimeOffset? ParseBroadcomTimestamp(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + if (DateTimeOffset.TryParse( + value, + CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, + out var parsed)) + { + return parsed.ToUniversalTime(); + } + + if (DateTime.TryParse( + value, + CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, + out var parsedDateTime)) + { + return new DateTimeOffset(parsedDateTime, TimeSpan.Zero).ToUniversalTime(); + } + + return null; + } + + private static string NormalizeVmwareAdvisoryId(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return string.Empty; + } + + var match = Regex.Match(value, "VMSA-\\d{4}-\\d{4}(?:\\.\\d+)?", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + if (!match.Success) + { + return string.Empty; + } + + var advisoryId = match.Value.ToUpperInvariant(); + var revisionSeparator = advisoryId.IndexOf('.', StringComparison.Ordinal); + return revisionSeparator >= 0 ? advisoryId[..revisionSeparator] : advisoryId; + } + private async Task GetCursorAsync(CancellationToken cancellationToken) { var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareServiceCollectionExtensions.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareServiceCollectionExtensions.cs index ce1fddaf7..f2968a2ba 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareServiceCollectionExtensions.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareServiceCollectionExtensions.cs @@ -27,7 +27,7 @@ public static class VmwareServiceCollectionExtensions clientOptions.UserAgent = "StellaOps.Concelier.VMware/1.0"; clientOptions.AllowedHosts.Clear(); clientOptions.AllowedHosts.Add(options.IndexUri.Host); - clientOptions.DefaultRequestHeaders["Accept"] = "application/json"; + clientOptions.DefaultRequestHeaders["Accept"] = "text/html, application/json"; }); services.TryAddSingleton(); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Observations/AdvisoryObservationEventPublisherOptions.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Observations/AdvisoryObservationEventPublisherOptions.cs index 49747c466..5fbfd1736 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Observations/AdvisoryObservationEventPublisherOptions.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Observations/AdvisoryObservationEventPublisherOptions.cs @@ -3,7 +3,7 @@ namespace StellaOps.Concelier.Core.Observations; public sealed class AdvisoryObservationEventPublisherOptions { public bool Enabled { get; set; } = false; - public string Transport { get; set; } = "inmemory"; // inmemory|nats + public string Transport { get; set; } = "postgres"; // postgres|nats public string? NatsUrl { get; set; } public string Subject { get; set; } = "concelier.advisory.observation.updated.v1"; public string DeadLetterSubject { get; set; } = "concelier.advisory.observation.updated.dead.v1"; diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceConnectivityResult.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceConnectivityResult.cs index 63a66880f..de9a981dc 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceConnectivityResult.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceConnectivityResult.cs @@ -6,6 +6,7 @@ // ----------------------------------------------------------------------------- using System.Collections.Immutable; +using System.Text.Json.Serialization; namespace StellaOps.Concelier.Core.Sources; @@ -157,6 +158,7 @@ public sealed record SourceConnectivityResult /// /// Connectivity status for a source. /// +[JsonConverter(typeof(JsonStringEnumConverter))] public enum SourceConnectivityStatus { /// Source is unknown or not checked. @@ -212,6 +214,7 @@ public sealed record RemediationStep /// /// Type of remediation command. /// +[JsonConverter(typeof(JsonStringEnumConverter))] public enum CommandType { /// Bash/shell command. diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceDefinitions.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceDefinitions.cs index 43ca30b63..ead0a51d6 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceDefinitions.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceDefinitions.cs @@ -313,7 +313,8 @@ public static class SourceDefinitions BaseEndpoint = "https://api.msrc.microsoft.com/sug/v2.0/en-US/", HealthCheckEndpoint = "https://api.msrc.microsoft.com/sug/v2.0/en-US/affectedProduct", HttpClientName = "MsrcClient", - RequiresAuthentication = false, + RequiresAuthentication = true, + CredentialEnvVar = "CONCELIER__SOURCES__VNDR__MSRC__TENANTID, CONCELIER__SOURCES__VNDR__MSRC__CLIENTID, CONCELIER__SOURCES__VNDR__MSRC__CLIENTSECRET", StatusPageUrl = "https://msrc.microsoft.com/", DocumentationUrl = "https://msrc.microsoft.com/update-guide/", DefaultPriority = 35, @@ -366,6 +367,21 @@ public static class SourceDefinitions Tags = ImmutableArray.Create("oracle", "vendor", "java") }; + public static readonly SourceDefinition Adobe = new() + { + Id = "adobe", + DisplayName = "Adobe Security", + Category = SourceCategory.Vendor, + Type = SourceType.Upstream, + Description = "Adobe security bulletins and advisories", + BaseEndpoint = "https://helpx.adobe.com/security/security-bulletin.html", + HealthCheckEndpoint = "https://helpx.adobe.com/security/security-bulletin.html", + HttpClientName = "source-vndr-adobe", + RequiresAuthentication = false, + DefaultPriority = 52, + Tags = ImmutableArray.Create("adobe", "vendor", "creative-cloud") + }; + public static readonly SourceDefinition Apple = new() { Id = "apple", @@ -381,6 +397,21 @@ public static class SourceDefinitions Tags = ImmutableArray.Create("apple", "vendor", "macos", "ios") }; + public static readonly SourceDefinition Chromium = new() + { + Id = "chromium", + DisplayName = "Chromium Security", + Category = SourceCategory.Vendor, + Type = SourceType.Upstream, + Description = "Chromium and Chrome stable channel security updates", + BaseEndpoint = "https://chromereleases.googleblog.com/atom.xml", + HealthCheckEndpoint = "https://chromereleases.googleblog.com/atom.xml", + HttpClientName = "source-vndr-chromium", + RequiresAuthentication = false, + DefaultPriority = 57, + Tags = ImmutableArray.Create("chromium", "vendor", "browser", "google") + }; + public static readonly SourceDefinition Cisco = new() { Id = "cisco", @@ -391,7 +422,8 @@ public static class SourceDefinitions BaseEndpoint = "https://tools.cisco.com/security/center/publicationService.x", HealthCheckEndpoint = "https://tools.cisco.com/security/center/publicationListing.x", HttpClientName = "CiscoClient", - RequiresAuthentication = false, + RequiresAuthentication = true, + CredentialEnvVar = "CONCELIER__SOURCES__VNDR__CISCO__CLIENTID, CONCELIER__SOURCES__VNDR__CISCO__CLIENTSECRET", StatusPageUrl = "https://status.cisco.com/", DefaultPriority = 60, Tags = ImmutableArray.Create("cisco", "vendor", "network") @@ -889,6 +921,22 @@ public static class SourceDefinitions Tags = ImmutableArray.Create("cert", "eu") }; + public static readonly SourceDefinition Cccs = new() + { + Id = "cccs", + DisplayName = "CCCS (Canada)", + Category = SourceCategory.Cert, + Type = SourceType.Upstream, + Description = "Canadian Centre for Cyber Security advisories", + BaseEndpoint = "https://www.cyber.gc.ca/api/cccs/threats/v1/get", + HealthCheckEndpoint = "https://www.cyber.gc.ca/en/", + HttpClientName = "concelier.source.cccs", + RequiresAuthentication = false, + Regions = ImmutableArray.Create("CA", "NA"), + DefaultPriority = 91, + Tags = ImmutableArray.Create("cert", "canada", "na") + }; + public static readonly SourceDefinition JpCert = new() { Id = "jpcert", @@ -905,6 +953,22 @@ public static class SourceDefinitions Tags = ImmutableArray.Create("cert", "japan", "apac") }; + public static readonly SourceDefinition CertCc = new() + { + Id = "cert-cc", + DisplayName = "CERT/CC", + Category = SourceCategory.Cert, + Type = SourceType.Upstream, + Description = "Carnegie Mellon CERT Coordination Center vulnerability notes", + BaseEndpoint = "https://www.kb.cert.org/vuls/api/", + HealthCheckEndpoint = "https://www.kb.cert.org/vulfeed", + HttpClientName = "certcc", + RequiresAuthentication = false, + Regions = ImmutableArray.Create("US", "NA"), + DefaultPriority = 93, + Tags = ImmutableArray.Create("cert", "usa", "na", "vu") + }; + public static readonly SourceDefinition UsCert = new() { Id = "us-cert", @@ -1407,7 +1471,7 @@ public static class SourceDefinitions // Primary databases Nvd, Osv, Ghsa, Cve, Epss, Kev, // Vendor advisories - RedHat, Microsoft, Amazon, Google, Oracle, Apple, Cisco, Fortinet, Juniper, Palo, Vmware, + RedHat, Microsoft, Amazon, Google, Oracle, Adobe, Apple, Chromium, Cisco, Fortinet, Juniper, Palo, Vmware, // Cloud provider advisories Aws, Azure, Gcp, // Linux distributions @@ -1427,7 +1491,7 @@ public static class SourceDefinitions // ICS/SCADA Siemens, KasperskyIcs, // CERTs - CertFr, CertDe, CertAt, CertBe, CertCh, CertEu, JpCert, UsCert, + CertFr, CertDe, CertAt, CertBe, CertCh, CertEu, Cccs, JpCert, CertCc, UsCert, // Additional CERTs CertUa, CertPl, AusCert, KrCert, CertIn, // Russian/CIS @@ -1468,5 +1532,5 @@ public static class SourceDefinitions /// Find a source by ID. /// public static SourceDefinition? FindById(string sourceId) - => All.FirstOrDefault(s => s.Id.Equals(sourceId, StringComparison.OrdinalIgnoreCase)); + => All.FirstOrDefault(s => s.Id.Equals(SourceKeyAliases.Normalize(sourceId), StringComparison.OrdinalIgnoreCase)); } diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceKeyAliases.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceKeyAliases.cs new file mode 100644 index 000000000..7acde89e5 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceKeyAliases.cs @@ -0,0 +1,94 @@ +using System.Collections.Immutable; + +namespace StellaOps.Concelier.Core.Sources; + +/// +/// Normalizes legacy connector source keys to the operator-facing source IDs exposed by +/// the catalog, scheduler, and UI surfaces. +/// +public static class SourceKeyAliases +{ + private static readonly ImmutableDictionary AliasToCanonical = + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["acsc"] = "auscert", + ["cert-bund"] = "cert-de", + ["certbund"] = "cert-de", + ["certfr"] = "cert-fr", + ["certin"] = "cert-in", + ["distro-alpine"] = "alpine", + ["distro-astra"] = "astra", + ["distro-debian"] = "debian", + ["distro-suse"] = "suse", + ["distro-ubuntu"] = "ubuntu", + ["ics-cisa"] = "us-cert", + ["ics-kaspersky"] = "kaspersky-ics", + ["jvn"] = "jpcert", + ["kisa"] = "krcert", + ["msrc"] = "microsoft", + ["ru-bdu"] = "fstec-bdu", + ["ru-nkcki"] = "nkcki", + ["vndr-adobe"] = "adobe", + ["vndr-apple"] = "apple", + ["vndr-cisco"] = "cisco", + ["vndr-chromium"] = "chromium", + ["vndr-msrc"] = "microsoft", + ["vndr-oracle"] = "oracle", + ["vndr.msrc"] = "microsoft", + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + private static readonly ImmutableDictionary> CanonicalToAliases = + AliasToCanonical + .GroupBy(static pair => pair.Value, StringComparer.OrdinalIgnoreCase) + .ToImmutableDictionary( + static group => group.Key, + static group => group + .Select(static pair => pair.Key) + .OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(), + StringComparer.OrdinalIgnoreCase); + + public static string Normalize(string? sourceKey) + { + if (string.IsNullOrWhiteSpace(sourceKey)) + { + return string.Empty; + } + + var trimmed = sourceKey.Trim(); + return AliasToCanonical.TryGetValue(trimmed, out var canonical) + ? canonical + : trimmed; + } + + public static ImmutableArray GetEquivalentKeys(string? sourceKey) + { + var normalized = Normalize(sourceKey); + if (string.IsNullOrWhiteSpace(normalized)) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + builder.Add(normalized); + + if (CanonicalToAliases.TryGetValue(normalized, out var aliases)) + { + builder.AddRange(aliases); + } + + var trimmed = sourceKey?.Trim(); + if (!string.IsNullOrWhiteSpace(trimmed) && + !builder.Any(candidate => string.Equals(candidate, trimmed, StringComparison.OrdinalIgnoreCase))) + { + builder.Add(trimmed); + } + + return builder + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + } + + public static bool Matches(string? left, string? right) + => string.Equals(Normalize(left), Normalize(right), StringComparison.OrdinalIgnoreCase); +} diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceRegistry.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceRegistry.cs index ca4f1d92e..4e16e76a8 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceRegistry.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceRegistry.cs @@ -65,7 +65,8 @@ public sealed class SourceRegistry : ISourceRegistry public SourceDefinition? GetSource(string sourceId) { ArgumentException.ThrowIfNullOrWhiteSpace(sourceId); - return _sources.FirstOrDefault(s => s.Id.Equals(sourceId, StringComparison.OrdinalIgnoreCase)); + var normalizedSourceId = SourceKeyAliases.Normalize(sourceId); + return _sources.FirstOrDefault(s => s.Id.Equals(normalizedSourceId, StringComparison.OrdinalIgnoreCase)); } /// @@ -77,11 +78,12 @@ public sealed class SourceRegistry : ISourceRegistry string sourceId, CancellationToken cancellationToken = default) { - var source = GetSource(sourceId); + var normalizedSourceId = SourceKeyAliases.Normalize(sourceId); + var source = GetSource(normalizedSourceId); if (source is null) { - var notFound = SourceConnectivityResult.NotFound(sourceId); - _lastCheckResults[sourceId] = notFound; + var notFound = SourceConnectivityResult.NotFound(string.IsNullOrWhiteSpace(normalizedSourceId) ? sourceId : normalizedSourceId); + _lastCheckResults[notFound.SourceId] = notFound; return notFound; } @@ -100,7 +102,7 @@ public sealed class SourceRegistry : ISourceRegistry _logger.LogDebug( "Checking connectivity for source {SourceId} at {Endpoint}", - sourceId, source.HealthCheckEndpoint); + normalizedSourceId, source.HealthCheckEndpoint); var response = await client.GetAsync( source.HealthCheckEndpoint, @@ -111,12 +113,12 @@ public sealed class SourceRegistry : ISourceRegistry if (response.IsSuccessStatusCode) { - var result = SourceConnectivityResult.Healthy(sourceId, stopwatch.Elapsed, checkedAt); - _lastCheckResults[sourceId] = result; + var result = SourceConnectivityResult.Healthy(normalizedSourceId, stopwatch.Elapsed, checkedAt); + _lastCheckResults[normalizedSourceId] = result; _logger.LogInformation( "Source {SourceId} is healthy (latency: {Latency}ms)", - sourceId, stopwatch.Elapsed.TotalMilliseconds); + normalizedSourceId, stopwatch.Elapsed.TotalMilliseconds); return result; } @@ -125,7 +127,7 @@ public sealed class SourceRegistry : ISourceRegistry var errorDetails = SourceErrorFactory.FromHttpResponse(source, response.StatusCode); var failedResult = SourceConnectivityResult.Failed( - sourceId, + normalizedSourceId, errorDetails.Code, errorDetails.Message, errorDetails.PossibleReasons, @@ -134,11 +136,11 @@ public sealed class SourceRegistry : ISourceRegistry stopwatch.Elapsed, (int)response.StatusCode); - _lastCheckResults[sourceId] = failedResult; + _lastCheckResults[normalizedSourceId] = failedResult; _logger.LogWarning( "Source {SourceId} failed connectivity check: {StatusCode} - {ErrorMessage}", - sourceId, response.StatusCode, errorDetails.Message); + normalizedSourceId, response.StatusCode, errorDetails.Message); return failedResult; } @@ -148,7 +150,7 @@ public sealed class SourceRegistry : ISourceRegistry var errorDetails = SourceErrorFactory.FromNetworkException(source, ex); var failedResult = SourceConnectivityResult.Failed( - sourceId, + normalizedSourceId, errorDetails.Code, errorDetails.Message, errorDetails.PossibleReasons, @@ -156,11 +158,11 @@ public sealed class SourceRegistry : ISourceRegistry checkedAt, stopwatch.Elapsed); - _lastCheckResults[sourceId] = failedResult; + _lastCheckResults[normalizedSourceId] = failedResult; _logger.LogWarning(ex, "Source {SourceId} failed connectivity check (network error): {ErrorMessage}", - sourceId, errorDetails.Message); + normalizedSourceId, errorDetails.Message); return failedResult; } @@ -174,7 +176,7 @@ public sealed class SourceRegistry : ISourceRegistry var errorDetails = SourceErrorFactory.FromNetworkException(source, new TaskCanceledException()); var failedResult = SourceConnectivityResult.Failed( - sourceId, + normalizedSourceId, errorDetails.Code, errorDetails.Message, errorDetails.PossibleReasons, @@ -182,11 +184,11 @@ public sealed class SourceRegistry : ISourceRegistry checkedAt, stopwatch.Elapsed); - _lastCheckResults[sourceId] = failedResult; + _lastCheckResults[normalizedSourceId] = failedResult; _logger.LogWarning( "Source {SourceId} connectivity check timed out after {Timeout}s", - sourceId, _configuration.ConnectivityCheckTimeoutSeconds); + normalizedSourceId, _configuration.ConnectivityCheckTimeoutSeconds); return failedResult; } @@ -194,7 +196,7 @@ public sealed class SourceRegistry : ISourceRegistry { stopwatch.Stop(); var failedResult = SourceConnectivityResult.Failed( - sourceId, + normalizedSourceId, "UNEXPECTED_ERROR", $"Unexpected error: {ex.Message}", ImmutableArray.Create("An unexpected error occurred during connectivity check"), @@ -207,11 +209,11 @@ public sealed class SourceRegistry : ISourceRegistry checkedAt, stopwatch.Elapsed); - _lastCheckResults[sourceId] = failedResult; + _lastCheckResults[normalizedSourceId] = failedResult; _logger.LogError(ex, "Source {SourceId} connectivity check failed with unexpected error", - sourceId); + normalizedSourceId); return failedResult; } @@ -284,40 +286,41 @@ public sealed class SourceRegistry : ISourceRegistry CancellationToken cancellationToken = default) { ArgumentException.ThrowIfNullOrWhiteSpace(sourceId); + var normalizedSourceId = SourceKeyAliases.Normalize(sourceId); - var source = GetSource(sourceId); + var source = GetSource(normalizedSourceId); if (source is null) { _logger.LogWarning("Attempted to enable unknown source: {SourceId}", sourceId); return false; } - _enabledSources[sourceId] = true; - _logger.LogInformation("Enabled source: {SourceId}", sourceId); + _enabledSources[normalizedSourceId] = true; + _logger.LogInformation("Enabled source: {SourceId}", normalizedSourceId); // Auto-trigger initial sync pipeline on enable. if (_sourceSyncTrigger is not null) { try { - var result = await _sourceSyncTrigger.TriggerAsync(sourceId, "source-enable", cancellationToken).ConfigureAwait(false); + var result = await _sourceSyncTrigger.TriggerAsync(normalizedSourceId, "source-enable", cancellationToken).ConfigureAwait(false); if (result.Outcome == JobTriggerOutcome.Accepted) { - _logger.LogInformation("Auto-triggered sync pipeline for source {SourceId} on enable", sourceId); + _logger.LogInformation("Auto-triggered sync pipeline for source {SourceId} on enable", normalizedSourceId); } else { - _logger.LogDebug("Sync pipeline for source {SourceId} was not started on enable: {Outcome}", sourceId, result.Outcome); + _logger.LogDebug("Sync pipeline for source {SourceId} was not started on enable: {Outcome}", normalizedSourceId, result.Outcome); } } catch (Exception ex) { - _logger.LogWarning(ex, "Failed to auto-trigger sync pipeline for source {SourceId} on enable", sourceId); + _logger.LogWarning(ex, "Failed to auto-trigger sync pipeline for source {SourceId} on enable", normalizedSourceId); } } else if (_jobCoordinator is not null) { - var fetchKind = $"source:{sourceId}:fetch"; + var fetchKind = $"source:{normalizedSourceId}:fetch"; try { var result = await _jobCoordinator.TriggerAsync(fetchKind, null, "source-enable", cancellationToken); @@ -345,16 +348,17 @@ public sealed class SourceRegistry : ISourceRegistry CancellationToken cancellationToken = default) { ArgumentException.ThrowIfNullOrWhiteSpace(sourceId); + var normalizedSourceId = SourceKeyAliases.Normalize(sourceId); - var source = GetSource(sourceId); + var source = GetSource(normalizedSourceId); if (source is null) { _logger.LogWarning("Attempted to disable unknown source: {SourceId}", sourceId); return Task.FromResult(false); } - _enabledSources[sourceId] = false; - _logger.LogInformation("Disabled source: {SourceId}", sourceId); + _enabledSources[normalizedSourceId] = false; + _logger.LogInformation("Disabled source: {SourceId}", normalizedSourceId); return Task.FromResult(true); } @@ -384,7 +388,7 @@ public sealed class SourceRegistry : ISourceRegistry /// public SourceConnectivityResult? GetLastCheckResult(string sourceId) { - return _lastCheckResults.GetValueOrDefault(sourceId); + return _lastCheckResults.GetValueOrDefault(SourceKeyAliases.Normalize(sourceId)); } /// @@ -392,7 +396,7 @@ public sealed class SourceRegistry : ISourceRegistry /// public bool IsEnabled(string sourceId) { - return _enabledSources.GetValueOrDefault(sourceId); + return _enabledSources.GetValueOrDefault(SourceKeyAliases.Normalize(sourceId)); } private static void ConfigureClientHeaders(HttpClient client, SourceDefinition source) diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Extensions/ConcelierPersistenceExtensions.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Extensions/ConcelierPersistenceExtensions.cs index ee9d2bed4..a57f1c257 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Extensions/ConcelierPersistenceExtensions.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Extensions/ConcelierPersistenceExtensions.cs @@ -5,9 +5,12 @@ using HistoryContracts = StellaOps.Concelier.Storage.ChangeHistory; using JpFlagsContracts = StellaOps.Concelier.Storage.JpFlags; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using PsirtContracts = StellaOps.Concelier.Storage.PsirtFlags; +using StellaOps.Concelier.Core.Jobs; using StellaOps.Concelier.Core.Linksets; using StellaOps.Concelier.Core.Observations; +using StellaOps.Concelier.Core.Orchestration; using StellaOps.Concelier.Core.Signals; using StellaOps.Concelier.Merge.Backport; using StellaOps.Concelier.Persistence.Postgres; @@ -37,6 +40,7 @@ public static class ConcelierPersistenceExtensions string sectionName = "Postgres:Concelier") { services.Configure(sectionName, configuration.GetSection(sectionName)); + services.TryAddSingleton(TimeProvider.System); services.AddSingleton(); services.AddStartupMigrations( ConcelierDataSource.DefaultSchemaName, @@ -76,6 +80,11 @@ public static class ConcelierPersistenceExtensions services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(); // Provenance scope services (backport integration) services.AddScoped(); @@ -95,6 +104,7 @@ public static class ConcelierPersistenceExtensions Action configureOptions) { services.Configure(configureOptions); + services.TryAddSingleton(TimeProvider.System); services.AddSingleton(); services.AddStartupMigrations( ConcelierDataSource.DefaultSchemaName, @@ -134,6 +144,11 @@ public static class ConcelierPersistenceExtensions services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(); // Provenance scope services (backport integration) services.AddScoped(); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Migrations/006_fix_advisory_source_signature_projection_counts.sql b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Migrations/006_fix_advisory_source_signature_projection_counts.sql new file mode 100644 index 000000000..8b75bfa64 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Migrations/006_fix_advisory_source_signature_projection_counts.sql @@ -0,0 +1,60 @@ +-- Concelier migration 006: fix advisory-source signature projection counts +-- Sprint: SPRINT_20260421_001 + +CREATE OR REPLACE VIEW vuln.advisory_source_signature_projection AS +WITH advisory_totals AS ( + SELECT + e.source_id, + COUNT(DISTINCT e.source_advisory_id)::BIGINT AS total_advisories + FROM vuln.advisory_source_edge e + WHERE NULLIF(BTRIM(e.source_advisory_id), '') IS NOT NULL + GROUP BY e.source_id +), +signed_totals AS ( + SELECT + e.source_id, + COUNT(DISTINCT e.source_advisory_id)::BIGINT AS signed_advisories + FROM vuln.advisory_source_edge e + WHERE NULLIF(BTRIM(e.source_advisory_id), '') IS NOT NULL + AND e.dsse_envelope IS NOT NULL + AND CASE + WHEN jsonb_typeof(e.dsse_envelope->'signatures') = 'array' + THEN jsonb_array_length(e.dsse_envelope->'signatures') > 0 + ELSE FALSE + END + GROUP BY e.source_id +), +failure_totals AS ( + SELECT + ss.source_id, + CASE + WHEN ss.metadata ? 'signature_failure_count' + AND (ss.metadata->>'signature_failure_count') ~ '^[0-9]+$' + THEN (ss.metadata->>'signature_failure_count')::BIGINT + ELSE 0::BIGINT + END AS signature_failure_count + FROM vuln.source_states ss +) +SELECT + s.id AS source_id, + COALESCE(t.total_advisories, 0)::BIGINT AS total_advisories, + LEAST( + COALESCE(t.total_advisories, 0)::BIGINT, + COALESCE(st.signed_advisories, 0)::BIGINT + ) AS signed_advisories, + GREATEST( + COALESCE(t.total_advisories, 0)::BIGINT + - LEAST( + COALESCE(t.total_advisories, 0)::BIGINT, + COALESCE(st.signed_advisories, 0)::BIGINT + ), + 0::BIGINT + ) AS unsigned_advisories, + COALESCE(f.signature_failure_count, 0)::BIGINT AS signature_failure_count +FROM vuln.sources s +LEFT JOIN advisory_totals t ON t.source_id = s.id +LEFT JOIN signed_totals st ON st.source_id = s.id +LEFT JOIN failure_totals f ON f.source_id = s.id; + +COMMENT ON VIEW vuln.advisory_source_signature_projection IS + 'Per-source advisory totals and signature rollups derived from distinct source documents in vuln.advisory_source_edge.'; diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Migrations/009_add_job_runs_and_orchestrator_runtime.sql b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Migrations/009_add_job_runs_and_orchestrator_runtime.sql new file mode 100644 index 000000000..99c5c0dd2 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Migrations/009_add_job_runs_and_orchestrator_runtime.sql @@ -0,0 +1,91 @@ +CREATE TABLE IF NOT EXISTS vuln.job_runs ( + run_id UUID PRIMARY KEY, + kind TEXT NOT NULL, + status TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + started_at TIMESTAMPTZ NULL, + completed_at TIMESTAMPTZ NULL, + trigger TEXT NOT NULL, + parameters_hash TEXT NULL, + error TEXT NULL, + timeout_ms BIGINT NULL CHECK (timeout_ms IS NULL OR timeout_ms >= 0), + lease_duration_ms BIGINT NULL CHECK (lease_duration_ms IS NULL OR lease_duration_ms >= 0), + parameters_json JSONB NOT NULL DEFAULT '{}'::jsonb +); + +CREATE INDEX IF NOT EXISTS idx_job_runs_kind_created + ON vuln.job_runs(kind, created_at DESC, run_id ASC); +CREATE INDEX IF NOT EXISTS idx_job_runs_status_created + ON vuln.job_runs(status, created_at DESC, run_id ASC); +CREATE INDEX IF NOT EXISTS idx_job_runs_created + ON vuln.job_runs(created_at DESC, run_id ASC); + +CREATE TABLE IF NOT EXISTS vuln.orchestrator_registry ( + tenant_id TEXT NOT NULL, + connector_id TEXT NOT NULL, + source TEXT NOT NULL, + capabilities TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[], + auth_ref TEXT NOT NULL, + schedule_json JSONB NOT NULL, + rate_policy_json JSONB NOT NULL, + artifact_kinds TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[], + lock_key TEXT NOT NULL, + egress_guard_json JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (tenant_id, connector_id) +); + +CREATE INDEX IF NOT EXISTS idx_orchestrator_registry_tenant_updated + ON vuln.orchestrator_registry(tenant_id, updated_at DESC, connector_id ASC); + +CREATE TABLE IF NOT EXISTS vuln.orchestrator_heartbeats ( + tenant_id TEXT NOT NULL, + connector_id TEXT NOT NULL, + run_id UUID NOT NULL, + sequence BIGINT NOT NULL, + status TEXT NOT NULL, + progress INTEGER NULL, + queue_depth INTEGER NULL, + last_artifact_hash TEXT NULL, + last_artifact_kind TEXT NULL, + error_code TEXT NULL, + retry_after_seconds INTEGER NULL, + timestamp_utc TIMESTAMPTZ NOT NULL, + PRIMARY KEY (tenant_id, connector_id, run_id, sequence) +); + +CREATE INDEX IF NOT EXISTS idx_orchestrator_heartbeats_latest + ON vuln.orchestrator_heartbeats(tenant_id, connector_id, run_id, sequence DESC); + +CREATE TABLE IF NOT EXISTS vuln.orchestrator_commands ( + tenant_id TEXT NOT NULL, + connector_id TEXT NOT NULL, + run_id UUID NOT NULL, + sequence BIGINT NOT NULL, + command TEXT NOT NULL, + throttle_json JSONB NULL, + backfill_json JSONB NULL, + created_at TIMESTAMPTZ NOT NULL, + expires_at TIMESTAMPTZ NULL, + PRIMARY KEY (tenant_id, connector_id, run_id, sequence) +); + +CREATE INDEX IF NOT EXISTS idx_orchestrator_commands_pending + ON vuln.orchestrator_commands(tenant_id, connector_id, run_id, sequence ASC); +CREATE INDEX IF NOT EXISTS idx_orchestrator_commands_expires + ON vuln.orchestrator_commands(expires_at); + +CREATE TABLE IF NOT EXISTS vuln.orchestrator_manifests ( + tenant_id TEXT NOT NULL, + connector_id TEXT NOT NULL, + run_id UUID NOT NULL, + cursor_range_json JSONB NOT NULL, + artifact_hashes TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[], + dsse_envelope_hash TEXT NULL, + completed_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (tenant_id, connector_id, run_id) +); + +CREATE INDEX IF NOT EXISTS idx_orchestrator_manifests_completed + ON vuln.orchestrator_manifests(tenant_id, connector_id, completed_at DESC, run_id ASC); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Migrations/010_add_advisory_source_content_counts.sql b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Migrations/010_add_advisory_source_content_counts.sql new file mode 100644 index 000000000..04874d5e6 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Migrations/010_add_advisory_source_content_counts.sql @@ -0,0 +1,74 @@ +-- Concelier migration 010: add truthful advisory-source content counts +-- Sprint: SPRINT_20260421_002 + +DROP VIEW IF EXISTS vuln.advisory_source_signature_projection; + +CREATE VIEW vuln.advisory_source_signature_projection AS +WITH content_totals AS ( + SELECT + e.source_id, + COUNT(DISTINCT e.source_advisory_id)::BIGINT AS source_document_count, + COUNT(DISTINCT e.canonical_id)::BIGINT AS canonical_advisory_count, + COUNT(DISTINCT NULLIF(BTRIM(c.cve), ''))::BIGINT AS cve_count, + COUNT(DISTINCT CASE + WHEN e.vendor_status IS NOT NULL + THEN e.source_advisory_id + ELSE NULL + END)::BIGINT AS vex_document_count + FROM vuln.advisory_source_edge e + LEFT JOIN vuln.advisory_canonical c ON c.id = e.canonical_id + WHERE NULLIF(BTRIM(e.source_advisory_id), '') IS NOT NULL + GROUP BY e.source_id +), +signed_totals AS ( + SELECT + e.source_id, + COUNT(DISTINCT e.source_advisory_id)::BIGINT AS signed_advisories + FROM vuln.advisory_source_edge e + WHERE NULLIF(BTRIM(e.source_advisory_id), '') IS NOT NULL + AND e.dsse_envelope IS NOT NULL + AND CASE + WHEN jsonb_typeof(e.dsse_envelope->'signatures') = 'array' + THEN jsonb_array_length(e.dsse_envelope->'signatures') > 0 + ELSE FALSE + END + GROUP BY e.source_id +), +failure_totals AS ( + SELECT + ss.source_id, + CASE + WHEN ss.metadata ? 'signature_failure_count' + AND (ss.metadata->>'signature_failure_count') ~ '^[0-9]+$' + THEN (ss.metadata->>'signature_failure_count')::BIGINT + ELSE 0::BIGINT + END AS signature_failure_count + FROM vuln.source_states ss +) +SELECT + s.id AS source_id, + COALESCE(t.source_document_count, 0)::BIGINT AS total_advisories, + LEAST( + COALESCE(t.source_document_count, 0)::BIGINT, + COALESCE(st.signed_advisories, 0)::BIGINT + ) AS signed_advisories, + GREATEST( + COALESCE(t.source_document_count, 0)::BIGINT + - LEAST( + COALESCE(t.source_document_count, 0)::BIGINT, + COALESCE(st.signed_advisories, 0)::BIGINT + ), + 0::BIGINT + ) AS unsigned_advisories, + COALESCE(f.signature_failure_count, 0)::BIGINT AS signature_failure_count, + COALESCE(t.source_document_count, 0)::BIGINT AS source_document_count, + COALESCE(t.canonical_advisory_count, 0)::BIGINT AS canonical_advisory_count, + COALESCE(t.cve_count, 0)::BIGINT AS cve_count, + COALESCE(t.vex_document_count, 0)::BIGINT AS vex_document_count +FROM vuln.sources s +LEFT JOIN content_totals t ON t.source_id = s.id +LEFT JOIN signed_totals st ON st.source_id = s.id +LEFT JOIN failure_totals f ON f.source_id = s.id; + +COMMENT ON VIEW vuln.advisory_source_signature_projection IS + 'Per-source advisory-source review totals: distinct source documents, canonicals, CVEs, VEX-style documents, and signature rollups.'; diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Advisories/PostgresAdvisoryStore.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Advisories/PostgresAdvisoryStore.cs index 6a5a046f0..1f8b135df 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Advisories/PostgresAdvisoryStore.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Advisories/PostgresAdvisoryStore.cs @@ -1,6 +1,7 @@ using AdvisoryContracts = StellaOps.Concelier.Storage.Advisories; using Microsoft.Extensions.Logging; +using StellaOps.Concelier.Core.Sources; using StellaOps.Concelier.Models; using StellaOps.Concelier.Persistence.Postgres.Conversion; using StellaOps.Concelier.Persistence.Postgres.Models; @@ -28,6 +29,7 @@ public sealed class PostgresAdvisoryStore : IPostgresAdvisoryStore, AdvisoryCont private readonly IAdvisoryCreditRepository _creditRepository; private readonly IAdvisoryWeaknessRepository _weaknessRepository; private readonly IKevFlagRepository _kevFlagRepository; + private readonly ISourceRepository _sourceRepository; private readonly AdvisoryConverter _converter; private readonly ILogger _logger; @@ -54,6 +56,7 @@ public sealed class PostgresAdvisoryStore : IPostgresAdvisoryStore, AdvisoryCont IAdvisoryCreditRepository creditRepository, IAdvisoryWeaknessRepository weaknessRepository, IKevFlagRepository kevFlagRepository, + ISourceRepository sourceRepository, TimeProvider? timeProvider, ILogger logger) { @@ -65,6 +68,7 @@ public sealed class PostgresAdvisoryStore : IPostgresAdvisoryStore, AdvisoryCont _creditRepository = creditRepository ?? throw new ArgumentNullException(nameof(creditRepository)); _weaknessRepository = weaknessRepository ?? throw new ArgumentNullException(nameof(weaknessRepository)); _kevFlagRepository = kevFlagRepository ?? throw new ArgumentNullException(nameof(kevFlagRepository)); + _sourceRepository = sourceRepository ?? throw new ArgumentNullException(nameof(sourceRepository)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _converter = new AdvisoryConverter(timeProvider ?? TimeProvider.System); } @@ -99,7 +103,7 @@ public sealed class PostgresAdvisoryStore : IPostgresAdvisoryStore, AdvisoryCont /// Task AdvisoryContracts.IAdvisoryStore.UpsertAsync(Advisory advisory, CancellationToken cancellationToken) - => UpsertAsync(advisory, sourceId: null, cancellationToken); + => UpsertWithResolvedSourceAsync(advisory, cancellationToken); /// public async Task FindAsync(string advisoryKey, CancellationToken cancellationToken) @@ -191,6 +195,160 @@ public sealed class PostgresAdvisoryStore : IPostgresAdvisoryStore, AdvisoryCont return _advisoryRepository.CountAsync(cancellationToken); } + private async Task UpsertWithResolvedSourceAsync(Advisory advisory, CancellationToken cancellationToken) + { + var sourceId = await ResolveSourceIdFromAdvisoryAsync(advisory, cancellationToken).ConfigureAwait(false); + await UpsertAsync(advisory, sourceId, cancellationToken).ConfigureAwait(false); + } + + private async Task ResolveSourceIdFromAdvisoryAsync(Advisory advisory, CancellationToken cancellationToken) + { + var candidates = EnumerateSourceKeys(advisory) + .Select(SourceKeyAliases.Normalize) + .Where(static sourceKey => !string.IsNullOrWhiteSpace(sourceKey)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + if (candidates.Length == 0) + { + _logger.LogDebug( + "Advisory {AdvisoryKey} has no provenance source; persisting without source binding", + advisory.AdvisoryKey); + return null; + } + + var preferredSource = advisory.Provenance + .Select(static provenance => SourceKeyAliases.Normalize(provenance.Source)) + .FirstOrDefault(static sourceKey => !string.IsNullOrWhiteSpace(sourceKey)) + ?? candidates[0]; + + if (candidates.Length > 1) + { + _logger.LogWarning( + "Advisory {AdvisoryKey} has multiple provenance source candidates [{Candidates}]; binding to {PreferredSource}", + advisory.AdvisoryKey, + string.Join(", ", candidates), + preferredSource); + } + + return await ResolveSourceIdAsync(preferredSource, cancellationToken).ConfigureAwait(false); + } + + private static IEnumerable EnumerateSourceKeys(Advisory advisory) + { + foreach (var provenance in advisory.Provenance) + { + yield return provenance.Source; + } + + foreach (var reference in advisory.References) + { + yield return reference.Provenance.Source; + } + + foreach (var package in advisory.AffectedPackages) + { + foreach (var provenance in package.Provenance) + { + yield return provenance.Source; + } + + foreach (var range in package.VersionRanges) + { + yield return range.Provenance.Source; + } + + foreach (var status in package.Statuses) + { + yield return status.Provenance.Source; + } + } + + foreach (var metric in advisory.CvssMetrics) + { + yield return metric.Provenance.Source; + } + + foreach (var weakness in advisory.Cwes) + { + foreach (var provenance in weakness.Provenance) + { + yield return provenance.Source; + } + } + + foreach (var credit in advisory.Credits) + { + yield return credit.Provenance.Source; + } + } + + private async Task ResolveSourceIdAsync(string sourceKey, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sourceKey); + + var normalizedSourceKey = SourceKeyAliases.Normalize(sourceKey); + var existing = await FindSourceAsync(normalizedSourceKey, cancellationToken).ConfigureAwait(false); + if (existing is not null) + { + if (string.Equals(existing.Key, normalizedSourceKey, StringComparison.OrdinalIgnoreCase)) + { + return existing.Id; + } + + var canonical = await _sourceRepository.UpsertAsync( + new SourceEntity + { + Id = Guid.NewGuid(), + Key = normalizedSourceKey, + Name = normalizedSourceKey, + SourceType = normalizedSourceKey, + Url = existing.Url, + Priority = existing.Priority, + Enabled = existing.Enabled, + Config = existing.Config, + Metadata = existing.Metadata, + CreatedAt = existing.CreatedAt, + UpdatedAt = existing.UpdatedAt + }, + cancellationToken) + .ConfigureAwait(false); + + return canonical.Id; + } + + var created = await _sourceRepository.UpsertAsync( + new SourceEntity + { + Id = Guid.NewGuid(), + Key = normalizedSourceKey, + Name = normalizedSourceKey, + SourceType = normalizedSourceKey, + Priority = 100, + Enabled = true, + Config = "{}", + Metadata = "{}" + }, + cancellationToken) + .ConfigureAwait(false); + + return created.Id; + } + + private async Task FindSourceAsync(string sourceKey, CancellationToken cancellationToken) + { + foreach (var candidate in SourceKeyAliases.GetEquivalentKeys(sourceKey)) + { + var existing = await _sourceRepository.GetByKeyAsync(candidate, cancellationToken).ConfigureAwait(false); + if (existing is not null) + { + return existing; + } + } + + return null; + } + /// /// Reconstructs an Advisory domain model from a PostgreSQL entity. /// diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/DocumentStore.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/DocumentStore.cs index b510c060b..8dfd36dd7 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/DocumentStore.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/DocumentStore.cs @@ -1,5 +1,6 @@ using Contracts = StellaOps.Concelier.Storage.Contracts; +using StellaOps.Concelier.Core.Sources; using StellaOps.Concelier.Persistence.Postgres.Models; using StellaOps.Concelier.Persistence.Postgres.Repositories; using StellaOps.Concelier.Storage; @@ -30,8 +31,16 @@ public sealed class PostgresDocumentStore : IDocumentStore, Contracts.IStorageDo public async Task FindBySourceAndUriAsync(string sourceName, string uri, CancellationToken cancellationToken) { - var row = await _repository.FindBySourceAndUriAsync(sourceName, uri, cancellationToken).ConfigureAwait(false); - return row is null ? null : Map(row); + foreach (var candidate in SourceKeyAliases.GetEquivalentKeys(sourceName)) + { + var row = await _repository.FindBySourceAndUriAsync(candidate, uri, cancellationToken).ConfigureAwait(false); + if (row is not null) + { + return Map(row); + } + } + + return null; } public async Task UpsertAsync(DocumentRecord record, CancellationToken cancellationToken) @@ -101,19 +110,37 @@ public sealed class PostgresDocumentStore : IDocumentStore, Contracts.IStorageDo private async Task EnsureSourceAsync(string sourceName, CancellationToken cancellationToken) { - var existing = await _sourceRepository.GetByKeyAsync(sourceName, cancellationToken).ConfigureAwait(false); + var normalizedSourceName = SourceKeyAliases.Normalize(sourceName); + var existing = await FindSourceAsync(normalizedSourceName, cancellationToken).ConfigureAwait(false); if (existing is not null) { - return existing; + return string.Equals(existing.Key, normalizedSourceName, StringComparison.OrdinalIgnoreCase) + ? existing + : await _sourceRepository.UpsertAsync( + new SourceEntity + { + Id = Guid.NewGuid(), + Key = normalizedSourceName, + Name = normalizedSourceName, + SourceType = normalizedSourceName, + Url = existing.Url, + Priority = existing.Priority, + Enabled = existing.Enabled, + Config = existing.Config, + Metadata = existing.Metadata, + CreatedAt = existing.CreatedAt, + UpdatedAt = DateTimeOffset.UtcNow, + }, + cancellationToken).ConfigureAwait(false); } var now = DateTimeOffset.UtcNow; return await _sourceRepository.UpsertAsync(new SourceEntity { Id = Guid.NewGuid(), - Key = sourceName, - Name = sourceName, - SourceType = sourceName, + Key = normalizedSourceName, + Name = normalizedSourceName, + SourceType = normalizedSourceName, Url = null, Priority = 0, Enabled = true, @@ -123,4 +150,18 @@ public sealed class PostgresDocumentStore : IDocumentStore, Contracts.IStorageDo UpdatedAt = now, }, cancellationToken).ConfigureAwait(false); } + + private async Task FindSourceAsync(string sourceName, CancellationToken cancellationToken) + { + foreach (var candidate in SourceKeyAliases.GetEquivalentKeys(sourceName)) + { + var source = await _sourceRepository.GetByKeyAsync(candidate, cancellationToken).ConfigureAwait(false); + if (source is not null) + { + return source; + } + } + + return null; + } } diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/PostgresCanonicalAdvisoryStore.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/PostgresCanonicalAdvisoryStore.cs index 924d0b815..a4b58753a 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/PostgresCanonicalAdvisoryStore.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/PostgresCanonicalAdvisoryStore.cs @@ -1,5 +1,6 @@ using System.Text.Json; using StellaOps.Concelier.Core.Canonical; +using StellaOps.Concelier.Core.Sources; using StellaOps.Concelier.Merge.Backport; using StellaOps.Concelier.Persistence.Postgres.Models; using StellaOps.Concelier.Persistence.Postgres.Repositories; @@ -166,20 +167,44 @@ public sealed class PostgresCanonicalAdvisoryStore : ICanonicalAdvisoryStore public async Task ResolveSourceIdAsync(string sourceKey, CancellationToken ct = default) { ArgumentException.ThrowIfNullOrWhiteSpace(sourceKey); + var normalizedSourceKey = SourceKeyAliases.Normalize(sourceKey); - var existing = await sourceRepository.GetByKeyAsync(sourceKey, ct).ConfigureAwait(false); + var existing = await FindSourceAsync(normalizedSourceKey, ct).ConfigureAwait(false); if (existing is not null) { - return existing.Id; + if (string.Equals(existing.Key, normalizedSourceKey, StringComparison.OrdinalIgnoreCase)) + { + return existing.Id; + } + + var canonical = await sourceRepository.UpsertAsync( + new SourceEntity + { + Id = Guid.NewGuid(), + Key = normalizedSourceKey, + Name = normalizedSourceKey, + SourceType = normalizedSourceKey, + Url = existing.Url, + Priority = existing.Priority, + Enabled = existing.Enabled, + Config = existing.Config, + Metadata = existing.Metadata, + CreatedAt = existing.CreatedAt, + UpdatedAt = existing.UpdatedAt + }, + ct) + .ConfigureAwait(false); + + return canonical.Id; } var created = await sourceRepository.UpsertAsync( new SourceEntity { Id = Guid.NewGuid(), - Key = sourceKey.Trim(), - Name = sourceKey.Trim(), - SourceType = sourceKey.Trim(), + Key = normalizedSourceKey, + Name = normalizedSourceKey, + SourceType = normalizedSourceKey, Priority = 100, Enabled = true, Config = "{}", @@ -194,11 +219,26 @@ public sealed class PostgresCanonicalAdvisoryStore : ICanonicalAdvisoryStore public async Task GetSourcePrecedenceAsync(string sourceKey, CancellationToken ct = default) { ArgumentException.ThrowIfNullOrWhiteSpace(sourceKey); + var normalizedSourceKey = SourceKeyAliases.Normalize(sourceKey); - var source = await sourceRepository.GetByKeyAsync(sourceKey, ct).ConfigureAwait(false); + var source = await FindSourceAsync(normalizedSourceKey, ct).ConfigureAwait(false); return source?.Priority ?? 100; } + private async Task FindSourceAsync(string sourceKey, CancellationToken ct) + { + foreach (var candidate in SourceKeyAliases.GetEquivalentKeys(sourceKey)) + { + var existing = await sourceRepository.GetByKeyAsync(candidate, ct).ConfigureAwait(false); + if (existing is not null) + { + return existing; + } + } + + return null; + } + private async Task> MapCanonicalsAsync( IReadOnlyList entities, CancellationToken ct) diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/AdvisorySourceReadRepository.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/AdvisorySourceReadRepository.cs index eb68807da..af65959bc 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/AdvisorySourceReadRepository.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/AdvisorySourceReadRepository.cs @@ -43,6 +43,10 @@ public sealed class AdvisorySourceReadRepository : RepositoryBase>'signature_status', 'unsigned') AS signature_status, + COALESCE(sig.source_document_count, 0) AS source_document_count, + COALESCE(sig.canonical_advisory_count, 0) AS canonical_advisory_count, + COALESCE(sig.cve_count, 0) AS cve_count, + COALESCE(sig.vex_document_count, 0) AS vex_document_count, COALESCE(sig.total_advisories, 0) AS total_advisories, COALESCE(sig.signed_advisories, 0) AS signed_advisories, COALESCE(sig.unsigned_advisories, 0) AS unsigned_advisories, @@ -79,6 +83,10 @@ public sealed class AdvisorySourceReadRepository : RepositoryBase>'signature_status', 'unsigned') AS signature_status, + COALESCE(sig.source_document_count, 0) AS source_document_count, + COALESCE(sig.canonical_advisory_count, 0) AS canonical_advisory_count, + COALESCE(sig.cve_count, 0) AS cve_count, + COALESCE(sig.vex_document_count, 0) AS vex_document_count, COALESCE(sig.total_advisories, 0) AS total_advisories, COALESCE(sig.signed_advisories, 0) AS signed_advisories, COALESCE(sig.unsigned_advisories, 0) AS unsigned_advisories, @@ -158,6 +170,10 @@ public sealed class AdvisorySourceReadRepository : RepositoryBase() : observation.Linkset.Aliases @@ -235,8 +237,83 @@ public sealed class PostgresAdvisoryObservationStore private static AdvisoryObservation MapObservation(NpgsqlDataReader reader) { var json = reader.GetString(0); - var observation = JsonSerializer.Deserialize(json, PostgresAffectedSymbolStore.SerializerOptions); - return observation ?? throw new InvalidOperationException("Failed to deserialize advisory observation payload."); + var payload = JsonSerializer.Deserialize(json, PostgresAffectedSymbolStore.SerializerOptions) + ?? throw new InvalidOperationException("Failed to deserialize advisory observation payload."); + + return new AdvisoryObservation( + payload.ObservationId, + payload.Tenant, + new AdvisoryObservationSource( + payload.Source.Vendor, + payload.Source.Stream, + payload.Source.Api, + payload.Source.CollectorVersion), + new AdvisoryObservationUpstream( + payload.Upstream.UpstreamId, + payload.Upstream.DocumentVersion, + payload.Upstream.FetchedAt, + payload.Upstream.ReceivedAt, + payload.Upstream.ContentHash, + new AdvisoryObservationSignature( + payload.Upstream.Signature.Present, + payload.Upstream.Signature.Format, + payload.Upstream.Signature.KeyId, + payload.Upstream.Signature.Signature), + payload.Upstream.Metadata?.ToImmutableDictionary(StringComparer.Ordinal)), + new AdvisoryObservationContent( + payload.Content.Format, + payload.Content.SpecVersion, + payload.Content.Raw, + payload.Content.Metadata?.ToImmutableDictionary(StringComparer.Ordinal)), + new AdvisoryObservationLinkset( + payload.Linkset.Aliases, + payload.Linkset.Purls, + payload.Linkset.Cpes, + payload.Linkset.References?.Select(reference => new AdvisoryObservationReference(reference.Type, reference.Url))), + payload.RawLinkset ?? new RawLinkset(), + payload.CreatedAt, + payload.Attributes?.ToImmutableDictionary(StringComparer.Ordinal)); + } + + private static string SerializeObservation(AdvisoryObservation observation) + { + var payload = new StoredObservationPayload( + observation.ObservationId, + observation.Tenant, + new StoredObservationSourcePayload( + observation.Source.Vendor, + observation.Source.Stream, + observation.Source.Api, + observation.Source.CollectorVersion), + new StoredObservationUpstreamPayload( + observation.Upstream.UpstreamId, + observation.Upstream.DocumentVersion, + observation.Upstream.FetchedAt, + observation.Upstream.ReceivedAt, + observation.Upstream.ContentHash, + new StoredObservationSignaturePayload( + observation.Upstream.Signature.Present, + observation.Upstream.Signature.Format, + observation.Upstream.Signature.KeyId, + observation.Upstream.Signature.Signature), + observation.Upstream.Metadata.ToDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.Ordinal)), + new StoredObservationContentPayload( + observation.Content.Format, + observation.Content.SpecVersion, + observation.Content.Raw, + observation.Content.Metadata.ToDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.Ordinal)), + new StoredObservationLinksetPayload( + observation.Linkset.Aliases.ToArray(), + observation.Linkset.Purls.ToArray(), + observation.Linkset.Cpes.ToArray(), + observation.Linkset.References + .Select(reference => new StoredObservationReferencePayload(reference.Type, reference.Url)) + .ToArray()), + observation.RawLinkset, + observation.CreatedAt, + observation.Attributes.ToDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.Ordinal)); + + return JsonSerializer.Serialize(payload, PostgresAffectedSymbolStore.SerializerOptions); } private static string[]? NormalizeValues(IEnumerable? values, bool lowerCase) @@ -294,4 +371,50 @@ public sealed class PostgresAdvisoryObservationStore UpstreamUrl: observation.Linkset.References.FirstOrDefault()?.Url) }; } + + private sealed record StoredObservationPayload( + string ObservationId, + string Tenant, + StoredObservationSourcePayload Source, + StoredObservationUpstreamPayload Upstream, + StoredObservationContentPayload Content, + StoredObservationLinksetPayload Linkset, + RawLinkset? RawLinkset, + DateTimeOffset CreatedAt, + Dictionary? Attributes); + + private sealed record StoredObservationSourcePayload( + string Vendor, + string Stream, + string Api, + string? CollectorVersion); + + private sealed record StoredObservationSignaturePayload( + bool Present, + string? Format, + string? KeyId, + string? Signature); + + private sealed record StoredObservationUpstreamPayload( + string UpstreamId, + string? DocumentVersion, + DateTimeOffset FetchedAt, + DateTimeOffset ReceivedAt, + string ContentHash, + StoredObservationSignaturePayload Signature, + Dictionary? Metadata); + + private sealed record StoredObservationContentPayload( + string Format, + string? SpecVersion, + JsonNode Raw, + Dictionary? Metadata); + + private sealed record StoredObservationReferencePayload(string Type, string Url); + + private sealed record StoredObservationLinksetPayload( + string[]? Aliases, + string[]? Purls, + string[]? Cpes, + StoredObservationReferencePayload[]? References); } diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/PostgresAffectedSymbolStore.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/PostgresAffectedSymbolStore.cs index 919937f41..87fde2cb8 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/PostgresAffectedSymbolStore.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/PostgresAffectedSymbolStore.cs @@ -49,9 +49,10 @@ public sealed class PostgresAffectedSymbolStore .ConfigureAwait(false); await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); - foreach (var observationGroup in tenantGroup.GroupBy( - symbol => (AdvisoryIdKey: NormalizeKey(symbol.AdvisoryId), ObservationId: NormalizeRequired(symbol.ObservationId)), - comparer: EqualityComparer<(string AdvisoryIdKey, string ObservationId)>.Default)) + foreach (var observationGroup in tenantGroup.GroupBy(symbol => new ObservationGroupKey( + NormalizeKey(symbol.AdvisoryId) + ?? throw new InvalidOperationException("Advisory ID key is required."), + NormalizeRequired(symbol.ObservationId)))) { await ReplaceObservationSymbolsAsync( connection, @@ -281,22 +282,22 @@ public sealed class PostgresAffectedSymbolStore bool includePaging) { AddParameter(command, "tenant_id", tenantId); - AddParameter(command, "advisory_id_key", NormalizeKey(options.AdvisoryId)); - AddParameter(command, "purl_key", NormalizeKey(options.Purl)); + AddOptionalTextParameter(command, "advisory_id_key", NormalizeKey(options.AdvisoryId)); + AddOptionalTextParameter(command, "purl_key", NormalizeKey(options.Purl)); command.Parameters.Add(new NpgsqlParameter("with_location_only", NpgsqlDbType.Boolean) { Value = options.WithLocationOnly.HasValue ? options.WithLocationOnly.Value : DBNull.Value }); - var symbolTypes = options.SymbolTypes is { HasValue: true } typed && !typed.Value.IsDefaultOrEmpty - ? typed.Value.Select(static value => value.ToString()).ToArray() + var symbolTypes = options.SymbolTypes is { } symbolTypeFilter && !symbolTypeFilter.IsDefaultOrEmpty + ? symbolTypeFilter.Select(static value => value.ToString()).ToArray() : null; AddTextArrayParameter(command, "symbol_types", symbolTypes); - var sources = options.Sources is { HasValue: true } sourceFilter && !sourceFilter.Value.IsDefaultOrEmpty - ? sourceFilter.Value - .Select(static source => NormalizeKey(source)) + var sources = options.Sources is { } sourceFilter && !sourceFilter.IsDefaultOrEmpty + ? sourceFilter + .Select(source => NormalizeKey(source)) .Where(static source => source is not null) .Select(static source => source!) .ToArray() @@ -403,4 +404,14 @@ public sealed class PostgresAffectedSymbolStore internal static string? NormalizeKey(string? value) => NormalizeNullable(value)?.ToLowerInvariant(); + + private static void AddOptionalTextParameter(NpgsqlCommand command, string name, string? value) + { + command.Parameters.Add(new NpgsqlParameter(name, NpgsqlDbType.Text) + { + TypedValue = value + }); + } + + private readonly record struct ObservationGroupKey(string AdvisoryIdKey, string ObservationId); } diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/PostgresJobStore.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/PostgresJobStore.cs new file mode 100644 index 000000000..3b9fcd8ab --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/PostgresJobStore.cs @@ -0,0 +1,397 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Npgsql; +using NpgsqlTypes; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Infrastructure.Postgres.Repositories; + +namespace StellaOps.Concelier.Persistence.Postgres.Repositories; + +/// +/// Durable PostgreSQL-backed job run store for Concelier scheduler history. +/// +public sealed class PostgresJobStore : RepositoryBase, IJobStore +{ + private const string SystemTenantId = "_system"; + + public PostgresJobStore(ConcelierDataSource dataSource, ILogger logger) + : base(dataSource, logger) + { + } + + public async Task CreateAsync(JobRunCreateRequest request, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + + const string sql = """ + INSERT INTO vuln.job_runs + (run_id, kind, status, created_at, started_at, completed_at, + trigger, parameters_hash, error, timeout_ms, lease_duration_ms, parameters_json) + VALUES + (@run_id, @kind, @status, @created_at, NULL, NULL, + @trigger, @parameters_hash, NULL, @timeout_ms, @lease_duration_ms, @parameters_json::jsonb) + ON CONFLICT (run_id) DO UPDATE SET + kind = EXCLUDED.kind, + status = EXCLUDED.status, + created_at = EXCLUDED.created_at, + started_at = EXCLUDED.started_at, + completed_at = EXCLUDED.completed_at, + trigger = EXCLUDED.trigger, + parameters_hash = EXCLUDED.parameters_hash, + error = EXCLUDED.error, + timeout_ms = EXCLUDED.timeout_ms, + lease_duration_ms = EXCLUDED.lease_duration_ms, + parameters_json = EXCLUDED.parameters_json + RETURNING run_id, kind, status, created_at, started_at, completed_at, + trigger, parameters_hash, error, timeout_ms, lease_duration_ms, parameters_json::text + """; + + var runId = ComputeDeterministicRunId(request); + var parametersJson = SerializeParameters(request.Parameters); + + await using var connection = await DataSource + .OpenConnectionAsync(SystemTenantId, "writer", cancellationToken) + .ConfigureAwait(false); + await using var command = CreateCommand(sql, connection); + AddParameter(command, "run_id", runId); + AddParameter(command, "kind", request.Kind.Trim()); + AddParameter(command, "status", JobRunStatus.Pending.ToString()); + AddParameter(command, "created_at", request.CreatedAt); + AddParameter(command, "trigger", request.Trigger.Trim()); + AddParameter(command, "parameters_hash", request.ParametersHash); + AddParameter(command, "timeout_ms", ToMilliseconds(request.Timeout)); + AddParameter(command, "lease_duration_ms", ToMilliseconds(request.LeaseDuration)); + command.Parameters.Add(new NpgsqlParameter("parameters_json", NpgsqlDbType.Jsonb) + { + TypedValue = parametersJson + }); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + throw new InvalidOperationException("PostgreSQL job store did not return the created run."); + } + + return MapSnapshot(reader); + } + + public async Task TryStartAsync(Guid runId, DateTimeOffset startedAt, CancellationToken cancellationToken) + { + const string sql = """ + UPDATE vuln.job_runs + SET status = @status, + started_at = @started_at + WHERE run_id = @run_id + RETURNING run_id, kind, status, created_at, started_at, completed_at, + trigger, parameters_hash, error, timeout_ms, lease_duration_ms, parameters_json::text + """; + + await using var connection = await DataSource + .OpenConnectionAsync(SystemTenantId, "writer", cancellationToken) + .ConfigureAwait(false); + await using var command = CreateCommand(sql, connection); + AddParameter(command, "run_id", runId); + AddParameter(command, "status", JobRunStatus.Running.ToString()); + AddParameter(command, "started_at", startedAt); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + return await reader.ReadAsync(cancellationToken).ConfigureAwait(false) + ? MapSnapshot(reader) + : null; + } + + public async Task TryCompleteAsync(Guid runId, JobRunCompletion completion, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(completion); + + const string sql = """ + UPDATE vuln.job_runs + SET status = @status, + completed_at = @completed_at, + error = @error + WHERE run_id = @run_id + RETURNING run_id, kind, status, created_at, started_at, completed_at, + trigger, parameters_hash, error, timeout_ms, lease_duration_ms, parameters_json::text + """; + + await using var connection = await DataSource + .OpenConnectionAsync(SystemTenantId, "writer", cancellationToken) + .ConfigureAwait(false); + await using var command = CreateCommand(sql, connection); + AddParameter(command, "run_id", runId); + AddParameter(command, "status", completion.Status.ToString()); + AddParameter(command, "completed_at", completion.CompletedAt); + AddParameter(command, "error", completion.Error); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + return await reader.ReadAsync(cancellationToken).ConfigureAwait(false) + ? MapSnapshot(reader) + : null; + } + + public Task FindAsync(Guid runId, CancellationToken cancellationToken) + { + const string sql = """ + SELECT run_id, kind, status, created_at, started_at, completed_at, + trigger, parameters_hash, error, timeout_ms, lease_duration_ms, parameters_json::text + FROM vuln.job_runs + WHERE run_id = @run_id + """; + + return QuerySingleOrDefaultAsync( + SystemTenantId, + sql, + command => AddParameter(command, "run_id", runId), + MapSnapshot, + cancellationToken); + } + + public Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken) + { + if (limit <= 0) + { + throw new ArgumentOutOfRangeException(nameof(limit)); + } + + const string sql = """ + SELECT run_id, kind, status, created_at, started_at, completed_at, + trigger, parameters_hash, error, timeout_ms, lease_duration_ms, parameters_json::text + FROM vuln.job_runs + WHERE @kind IS NULL OR kind = @kind + ORDER BY created_at DESC, run_id ASC + LIMIT @limit + """; + + var normalizedKind = string.IsNullOrWhiteSpace(kind) ? null : kind.Trim(); + + return QueryAsync( + SystemTenantId, + sql, + command => + { + AddParameter(command, "kind", normalizedKind); + AddParameter(command, "limit", limit); + }, + MapSnapshot, + cancellationToken); + } + + public Task> GetActiveRunsAsync(CancellationToken cancellationToken) + { + const string sql = """ + WITH ranked_active_runs AS ( + SELECT run_id, kind, status, created_at, started_at, completed_at, + trigger, parameters_hash, error, timeout_ms, lease_duration_ms, parameters_json::text, + ROW_NUMBER() OVER ( + PARTITION BY kind + ORDER BY CASE status + WHEN 'Running' THEN 0 + ELSE 1 + END ASC, + COALESCE(started_at, created_at) DESC, + created_at DESC, + run_id ASC + ) AS active_rank + FROM vuln.job_runs + WHERE status = ANY(@active_statuses) + AND EXISTS ( + SELECT 1 + FROM vuln.job_leases + WHERE lease_key = 'job:' || vuln.job_runs.kind + AND ttl_at > @now + ) + ) + SELECT run_id, kind, status, created_at, started_at, completed_at, + trigger, parameters_hash, error, timeout_ms, lease_duration_ms, parameters_json + FROM ranked_active_runs + WHERE active_rank = 1 + ORDER BY COALESCE(started_at, created_at) DESC, run_id ASC + """; + + return QueryAsync( + SystemTenantId, + sql, + command => + { + AddTextArrayParameter( + command, + "active_statuses", + [JobRunStatus.Pending.ToString(), JobRunStatus.Running.ToString()]); + AddParameter(command, "now", DateTimeOffset.UtcNow); + }, + MapSnapshot, + cancellationToken); + } + + public Task GetLastRunAsync(string kind, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(kind); + + const string sql = """ + SELECT run_id, kind, status, created_at, started_at, completed_at, + trigger, parameters_hash, error, timeout_ms, lease_duration_ms, parameters_json::text + FROM vuln.job_runs + WHERE kind = @kind + ORDER BY created_at DESC, run_id ASC + LIMIT 1 + """; + + return QuerySingleOrDefaultAsync( + SystemTenantId, + sql, + command => AddParameter(command, "kind", kind.Trim()), + MapSnapshot, + cancellationToken); + } + + public async Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(kinds); + + var requestedKinds = kinds + .Where(static kind => !string.IsNullOrWhiteSpace(kind)) + .Select(static kind => kind.Trim()) + .Distinct(StringComparer.Ordinal) + .ToArray(); + + if (requestedKinds.Length == 0) + { + return new Dictionary(StringComparer.Ordinal); + } + + const string sql = """ + SELECT DISTINCT ON (kind) + run_id, kind, status, created_at, started_at, completed_at, + trigger, parameters_hash, error, timeout_ms, lease_duration_ms, parameters_json::text + FROM vuln.job_runs + WHERE kind = ANY(@kinds) + ORDER BY kind ASC, created_at DESC, run_id ASC + """; + + var snapshots = await QueryAsync( + SystemTenantId, + sql, + command => AddTextArrayParameter(command, "kinds", requestedKinds), + MapSnapshot, + cancellationToken).ConfigureAwait(false); + + return snapshots.ToDictionary(snapshot => snapshot.Kind, StringComparer.Ordinal); + } + + private static JobRunSnapshot MapSnapshot(NpgsqlDataReader reader) + { + return new JobRunSnapshot( + reader.GetGuid(0), + reader.GetString(1), + Enum.Parse(reader.GetString(2), ignoreCase: false), + reader.GetFieldValue(3), + GetNullableDateTimeOffset(reader, 4), + GetNullableDateTimeOffset(reader, 5), + reader.GetString(6), + GetNullableString(reader, 7), + GetNullableString(reader, 8), + GetNullableInt64(reader, 9) is long timeoutMs ? TimeSpan.FromMilliseconds(timeoutMs) : null, + GetNullableInt64(reader, 10) is long leaseDurationMs ? TimeSpan.FromMilliseconds(leaseDurationMs) : null, + DeserializeParameters(reader.GetString(11))); + } + + private static Guid ComputeDeterministicRunId(JobRunCreateRequest request) + { + var input = $"job-run:{request.Kind}:{request.ParametersHash}:{request.CreatedAt:O}"; + var hashBytes = SHA256.HashData(Encoding.UTF8.GetBytes(input)); + return new Guid(hashBytes[..16]); + } + + private static long? ToMilliseconds(TimeSpan? duration) + { + if (!duration.HasValue) + { + return null; + } + + if (duration.Value < TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(duration), "Durations must be non-negative."); + } + + return checked((long)Math.Ceiling(duration.Value.TotalMilliseconds)); + } + + private static string SerializeParameters(IReadOnlyDictionary parameters) + { + ArgumentNullException.ThrowIfNull(parameters); + + var sorted = new SortedDictionary(StringComparer.Ordinal); + foreach (var pair in parameters) + { + sorted[pair.Key] = pair.Value; + } + + return JsonSerializer.Serialize(sorted, PostgresAffectedSymbolStore.SerializerOptions); + } + + private static IReadOnlyDictionary DeserializeParameters(string json) + { + if (string.IsNullOrWhiteSpace(json)) + { + return new Dictionary(StringComparer.Ordinal); + } + + using var document = JsonDocument.Parse(json); + if (document.RootElement.ValueKind != JsonValueKind.Object) + { + throw new InvalidOperationException("Persisted job parameters must be a JSON object."); + } + + var parameters = new Dictionary(StringComparer.Ordinal); + foreach (var property in document.RootElement.EnumerateObject()) + { + parameters[property.Name] = NormalizeJsonElement(property.Value); + } + + return parameters; + } + + private static object? NormalizeJsonElement(JsonElement element) + { + return element.ValueKind switch + { + JsonValueKind.Null => null, + JsonValueKind.String => element.GetString(), + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.Number => element.TryGetInt64(out var integer) + ? integer + : element.TryGetDecimal(out var decimalValue) + ? decimalValue + : element.GetDouble(), + JsonValueKind.Object => NormalizeJsonObject(element), + JsonValueKind.Array => NormalizeJsonArray(element), + _ => throw new InvalidOperationException($"Unsupported JSON value '{element.ValueKind}'."), + }; + } + + private static SortedDictionary NormalizeJsonObject(JsonElement element) + { + var values = new SortedDictionary(StringComparer.Ordinal); + foreach (var property in element.EnumerateObject()) + { + values[property.Name] = NormalizeJsonElement(property.Value); + } + + return values; + } + + private static List NormalizeJsonArray(JsonElement element) + { + var values = new List(); + foreach (var item in element.EnumerateArray()) + { + values.Add(NormalizeJsonElement(item)); + } + + return values; + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/PostgresOrchestratorRegistryStore.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/PostgresOrchestratorRegistryStore.cs new file mode 100644 index 000000000..7ac954876 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/PostgresOrchestratorRegistryStore.cs @@ -0,0 +1,427 @@ +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Npgsql; +using StellaOps.Concelier.Core.Orchestration; +using StellaOps.Infrastructure.Postgres.Repositories; + +namespace StellaOps.Concelier.Persistence.Postgres.Repositories; + +/// +/// Durable PostgreSQL-backed orchestrator registry store for connector runtime state. +/// +public sealed class PostgresOrchestratorRegistryStore + : RepositoryBase, + IOrchestratorRegistryStore +{ + private readonly TimeProvider _timeProvider; + + public PostgresOrchestratorRegistryStore( + ConcelierDataSource dataSource, + TimeProvider timeProvider, + ILogger logger) + : base(dataSource, logger) + { + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + } + + public Task UpsertAsync(OrchestratorRegistryRecord record, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(record); + + const string sql = """ + INSERT INTO vuln.orchestrator_registry + (tenant_id, connector_id, source, capabilities, auth_ref, + schedule_json, rate_policy_json, artifact_kinds, lock_key, + egress_guard_json, created_at, updated_at) + VALUES + (@tenant_id, @connector_id, @source, @capabilities, @auth_ref, + @schedule_json::jsonb, @rate_policy_json::jsonb, @artifact_kinds, @lock_key, + @egress_guard_json::jsonb, @created_at, @updated_at) + ON CONFLICT (tenant_id, connector_id) DO UPDATE SET + source = EXCLUDED.source, + capabilities = EXCLUDED.capabilities, + auth_ref = EXCLUDED.auth_ref, + schedule_json = EXCLUDED.schedule_json, + rate_policy_json = EXCLUDED.rate_policy_json, + artifact_kinds = EXCLUDED.artifact_kinds, + lock_key = EXCLUDED.lock_key, + egress_guard_json = EXCLUDED.egress_guard_json, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at + """; + + var tenant = PostgresAffectedSymbolStore.NormalizeTenant(record.Tenant); + return ExecuteAsync( + tenant, + sql, + command => + { + AddParameter(command, "tenant_id", tenant); + AddParameter(command, "connector_id", NormalizeRequired(record.ConnectorId)); + AddParameter(command, "source", NormalizeRequired(record.Source)); + AddTextArrayParameter(command, "capabilities", SanitizeTextList(record.Capabilities)); + AddParameter(command, "auth_ref", NormalizeRequired(record.AuthRef)); + AddJsonbParameter(command, "schedule_json", SerializeJson(record.Schedule)); + AddJsonbParameter(command, "rate_policy_json", SerializeJson(record.RatePolicy)); + AddTextArrayParameter(command, "artifact_kinds", SanitizeTextList(record.ArtifactKinds)); + AddParameter(command, "lock_key", NormalizeRequired(record.LockKey)); + AddJsonbParameter( + command, + "egress_guard_json", + SerializeJson(new OrchestratorEgressGuard( + SanitizeTextList(record.EgressGuard.Allowlist), + record.EgressGuard.AirgapMode))); + AddParameter(command, "created_at", record.CreatedAt); + AddParameter(command, "updated_at", record.UpdatedAt); + }, + cancellationToken); + } + + public Task GetAsync(string tenant, string connectorId, CancellationToken cancellationToken) + { + const string sql = """ + SELECT tenant_id, connector_id, source, capabilities, auth_ref, + schedule_json::text, rate_policy_json::text, artifact_kinds, lock_key, + egress_guard_json::text, created_at, updated_at + FROM vuln.orchestrator_registry + WHERE tenant_id = @tenant_id + AND connector_id = @connector_id + """; + + var normalizedTenant = PostgresAffectedSymbolStore.NormalizeTenant(tenant); + return QuerySingleOrDefaultAsync( + normalizedTenant, + sql, + command => + { + AddParameter(command, "tenant_id", normalizedTenant); + AddParameter(command, "connector_id", NormalizeRequired(connectorId)); + }, + MapRegistryRecord, + cancellationToken); + } + + public Task> ListAsync(string tenant, CancellationToken cancellationToken) + { + const string sql = """ + SELECT tenant_id, connector_id, source, capabilities, auth_ref, + schedule_json::text, rate_policy_json::text, artifact_kinds, lock_key, + egress_guard_json::text, created_at, updated_at + FROM vuln.orchestrator_registry + WHERE tenant_id = @tenant_id + ORDER BY connector_id ASC + """; + + var normalizedTenant = PostgresAffectedSymbolStore.NormalizeTenant(tenant); + return QueryAsync( + normalizedTenant, + sql, + command => AddParameter(command, "tenant_id", normalizedTenant), + MapRegistryRecord, + cancellationToken); + } + + public Task AppendHeartbeatAsync(OrchestratorHeartbeatRecord heartbeat, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(heartbeat); + + const string sql = """ + INSERT INTO vuln.orchestrator_heartbeats + (tenant_id, connector_id, run_id, sequence, status, progress, queue_depth, + last_artifact_hash, last_artifact_kind, error_code, retry_after_seconds, timestamp_utc) + VALUES + (@tenant_id, @connector_id, @run_id, @sequence, @status, @progress, @queue_depth, + @last_artifact_hash, @last_artifact_kind, @error_code, @retry_after_seconds, @timestamp_utc) + ON CONFLICT (tenant_id, connector_id, run_id, sequence) DO UPDATE SET + status = EXCLUDED.status, + progress = EXCLUDED.progress, + queue_depth = EXCLUDED.queue_depth, + last_artifact_hash = EXCLUDED.last_artifact_hash, + last_artifact_kind = EXCLUDED.last_artifact_kind, + error_code = EXCLUDED.error_code, + retry_after_seconds = EXCLUDED.retry_after_seconds, + timestamp_utc = EXCLUDED.timestamp_utc + """; + + var tenant = PostgresAffectedSymbolStore.NormalizeTenant(heartbeat.Tenant); + return ExecuteAsync( + tenant, + sql, + command => + { + AddParameter(command, "tenant_id", tenant); + AddParameter(command, "connector_id", NormalizeRequired(heartbeat.ConnectorId)); + AddParameter(command, "run_id", heartbeat.RunId); + AddParameter(command, "sequence", heartbeat.Sequence); + AddParameter(command, "status", heartbeat.Status.ToString()); + AddParameter(command, "progress", heartbeat.Progress); + AddParameter(command, "queue_depth", heartbeat.QueueDepth); + AddParameter(command, "last_artifact_hash", NormalizeNullable(heartbeat.LastArtifactHash)); + AddParameter(command, "last_artifact_kind", NormalizeNullable(heartbeat.LastArtifactKind)); + AddParameter(command, "error_code", NormalizeNullable(heartbeat.ErrorCode)); + AddParameter(command, "retry_after_seconds", heartbeat.RetryAfterSeconds); + AddParameter(command, "timestamp_utc", heartbeat.TimestampUtc); + }, + cancellationToken); + } + + public Task GetLatestHeartbeatAsync( + string tenant, + string connectorId, + Guid runId, + CancellationToken cancellationToken) + { + const string sql = """ + SELECT tenant_id, connector_id, run_id, sequence, status, progress, queue_depth, + last_artifact_hash, last_artifact_kind, error_code, retry_after_seconds, timestamp_utc + FROM vuln.orchestrator_heartbeats + WHERE tenant_id = @tenant_id + AND connector_id = @connector_id + AND run_id = @run_id + ORDER BY sequence DESC + LIMIT 1 + """; + + var normalizedTenant = PostgresAffectedSymbolStore.NormalizeTenant(tenant); + return QuerySingleOrDefaultAsync( + normalizedTenant, + sql, + command => + { + AddParameter(command, "tenant_id", normalizedTenant); + AddParameter(command, "connector_id", NormalizeRequired(connectorId)); + AddParameter(command, "run_id", runId); + }, + MapHeartbeatRecord, + cancellationToken); + } + + public Task EnqueueCommandAsync(OrchestratorCommandRecord command, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(command); + + const string sql = """ + INSERT INTO vuln.orchestrator_commands + (tenant_id, connector_id, run_id, sequence, command, + throttle_json, backfill_json, created_at, expires_at) + VALUES + (@tenant_id, @connector_id, @run_id, @sequence, @command, + @throttle_json::jsonb, @backfill_json::jsonb, @created_at, @expires_at) + ON CONFLICT (tenant_id, connector_id, run_id, sequence) DO UPDATE SET + command = EXCLUDED.command, + throttle_json = EXCLUDED.throttle_json, + backfill_json = EXCLUDED.backfill_json, + created_at = EXCLUDED.created_at, + expires_at = EXCLUDED.expires_at + """; + + var tenant = PostgresAffectedSymbolStore.NormalizeTenant(command.Tenant); + return ExecuteAsync( + tenant, + sql, + dbCommand => + { + AddParameter(dbCommand, "tenant_id", tenant); + AddParameter(dbCommand, "connector_id", NormalizeRequired(command.ConnectorId)); + AddParameter(dbCommand, "run_id", command.RunId); + AddParameter(dbCommand, "sequence", command.Sequence); + AddParameter(dbCommand, "command", command.Command.ToString()); + AddJsonbParameter(dbCommand, "throttle_json", command.Throttle is null ? null : SerializeJson(command.Throttle)); + AddJsonbParameter(dbCommand, "backfill_json", command.Backfill is null ? null : SerializeJson(command.Backfill)); + AddParameter(dbCommand, "created_at", command.CreatedAt); + AddParameter(dbCommand, "expires_at", command.ExpiresAt); + }, + cancellationToken); + } + + public Task> GetPendingCommandsAsync( + string tenant, + string connectorId, + Guid runId, + long? afterSequence, + CancellationToken cancellationToken) + { + const string sql = """ + SELECT tenant_id, connector_id, run_id, sequence, command, + throttle_json::text, backfill_json::text, created_at, expires_at + FROM vuln.orchestrator_commands + WHERE tenant_id = @tenant_id + AND connector_id = @connector_id + AND run_id = @run_id + AND (@after_sequence::bigint IS NULL OR sequence > @after_sequence::bigint) + AND (expires_at IS NULL OR expires_at > @now) + ORDER BY sequence ASC + """; + + var normalizedTenant = PostgresAffectedSymbolStore.NormalizeTenant(tenant); + return QueryAsync( + normalizedTenant, + sql, + command => + { + AddParameter(command, "tenant_id", normalizedTenant); + AddParameter(command, "connector_id", NormalizeRequired(connectorId)); + AddParameter(command, "run_id", runId); + var afterSequenceParameter = command.Parameters.Add("after_sequence", NpgsqlTypes.NpgsqlDbType.Bigint); + afterSequenceParameter.Value = afterSequence is null ? DBNull.Value : afterSequence.Value; + AddParameter(command, "now", _timeProvider.GetUtcNow()); + }, + MapCommandRecord, + cancellationToken); + } + + public Task StoreManifestAsync(OrchestratorRunManifest manifest, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(manifest); + + const string sql = """ + INSERT INTO vuln.orchestrator_manifests + (tenant_id, connector_id, run_id, cursor_range_json, + artifact_hashes, dsse_envelope_hash, completed_at) + VALUES + (@tenant_id, @connector_id, @run_id, @cursor_range_json::jsonb, + @artifact_hashes, @dsse_envelope_hash, @completed_at) + ON CONFLICT (tenant_id, connector_id, run_id) DO UPDATE SET + cursor_range_json = EXCLUDED.cursor_range_json, + artifact_hashes = EXCLUDED.artifact_hashes, + dsse_envelope_hash = EXCLUDED.dsse_envelope_hash, + completed_at = EXCLUDED.completed_at + """; + + var tenant = PostgresAffectedSymbolStore.NormalizeTenant(manifest.Tenant); + return ExecuteAsync( + tenant, + sql, + command => + { + AddParameter(command, "tenant_id", tenant); + AddParameter(command, "connector_id", NormalizeRequired(manifest.ConnectorId)); + AddParameter(command, "run_id", manifest.RunId); + AddJsonbParameter(command, "cursor_range_json", SerializeJson(manifest.CursorRange)); + AddTextArrayParameter(command, "artifact_hashes", SanitizeTextList(manifest.ArtifactHashes)); + AddParameter(command, "dsse_envelope_hash", NormalizeNullable(manifest.DsseEnvelopeHash)); + AddParameter(command, "completed_at", manifest.CompletedAt); + }, + cancellationToken); + } + + public Task GetManifestAsync( + string tenant, + string connectorId, + Guid runId, + CancellationToken cancellationToken) + { + const string sql = """ + SELECT tenant_id, connector_id, run_id, cursor_range_json::text, + artifact_hashes, dsse_envelope_hash, completed_at + FROM vuln.orchestrator_manifests + WHERE tenant_id = @tenant_id + AND connector_id = @connector_id + AND run_id = @run_id + """; + + var normalizedTenant = PostgresAffectedSymbolStore.NormalizeTenant(tenant); + return QuerySingleOrDefaultAsync( + normalizedTenant, + sql, + command => + { + AddParameter(command, "tenant_id", normalizedTenant); + AddParameter(command, "connector_id", NormalizeRequired(connectorId)); + AddParameter(command, "run_id", runId); + }, + MapManifestRecord, + cancellationToken); + } + + private static OrchestratorRegistryRecord MapRegistryRecord(NpgsqlDataReader reader) + { + return new OrchestratorRegistryRecord( + reader.GetString(0), + reader.GetString(1), + reader.GetString(2), + reader.GetFieldValue(3), + reader.GetString(4), + DeserializeJson(reader.GetString(5)), + DeserializeJson(reader.GetString(6)), + reader.GetFieldValue(7), + reader.GetString(8), + DeserializeJson(reader.GetString(9)), + reader.GetFieldValue(10), + reader.GetFieldValue(11)); + } + + private static OrchestratorHeartbeatRecord MapHeartbeatRecord(NpgsqlDataReader reader) + { + return new OrchestratorHeartbeatRecord( + reader.GetString(0), + reader.GetString(1), + reader.GetGuid(2), + reader.GetInt64(3), + Enum.Parse(reader.GetString(4), ignoreCase: false), + GetNullableInt32(reader, 5), + GetNullableInt32(reader, 6), + GetNullableString(reader, 7), + GetNullableString(reader, 8), + GetNullableString(reader, 9), + GetNullableInt32(reader, 10), + reader.GetFieldValue(11)); + } + + private static OrchestratorCommandRecord MapCommandRecord(NpgsqlDataReader reader) + { + return new OrchestratorCommandRecord( + reader.GetString(0), + reader.GetString(1), + reader.GetGuid(2), + reader.GetInt64(3), + Enum.Parse(reader.GetString(4), ignoreCase: false), + reader.IsDBNull(5) ? null : DeserializeJson(reader.GetString(5)), + reader.IsDBNull(6) ? null : DeserializeJson(reader.GetString(6)), + reader.GetFieldValue(7), + GetNullableDateTimeOffset(reader, 8)); + } + + private static OrchestratorRunManifest MapManifestRecord(NpgsqlDataReader reader) + { + return new OrchestratorRunManifest( + reader.GetGuid(2), + reader.GetString(1), + reader.GetString(0), + DeserializeJson(reader.GetString(3)), + reader.GetFieldValue(4), + GetNullableString(reader, 5), + reader.GetFieldValue(6)); + } + + private static string SerializeJson(T value) + where T : notnull + { + return JsonSerializer.Serialize(value, PostgresAffectedSymbolStore.SerializerOptions); + } + + private static T DeserializeJson(string json) + { + return JsonSerializer.Deserialize(json, PostgresAffectedSymbolStore.SerializerOptions) + ?? throw new InvalidOperationException($"Failed to deserialize orchestrator runtime payload '{typeof(T).Name}'."); + } + + private static string[] SanitizeTextList(IEnumerable? values) + { + if (values is null) + { + return Array.Empty(); + } + + return values + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Select(static value => value.Trim()) + .ToArray(); + } + + private static string NormalizeRequired(string value) + => PostgresAffectedSymbolStore.NormalizeRequired(value); + + private static string? NormalizeNullable(string? value) + => PostgresAffectedSymbolStore.NormalizeNullable(value); +} diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/ServiceCollectionExtensions.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/ServiceCollectionExtensions.cs index 132764f40..25ce6702a 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/ServiceCollectionExtensions.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/ServiceCollectionExtensions.cs @@ -5,11 +5,13 @@ using HistoryContracts = StellaOps.Concelier.Storage.ChangeHistory; using JpFlagsContracts = StellaOps.Concelier.Storage.JpFlags; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using PsirtContracts = StellaOps.Concelier.Storage.PsirtFlags; using StellaOps.Concelier.Core.Canonical; using StellaOps.Concelier.Core.Jobs; using StellaOps.Concelier.Core.Linksets; using StellaOps.Concelier.Core.Observations; +using StellaOps.Concelier.Core.Orchestration; using StellaOps.Concelier.Core.Signals; using StellaOps.Concelier.Merge.Backport; using StellaOps.Concelier.Persistence.Postgres.Advisories; @@ -40,6 +42,7 @@ public static class ServiceCollectionExtensions string sectionName = "Postgres:Concelier") { services.Configure(sectionName, configuration.GetSection(sectionName)); + services.TryAddSingleton(TimeProvider.System); services.AddSingleton(); services.AddStartupMigrations( ConcelierDataSource.DefaultSchemaName, @@ -85,6 +88,10 @@ public static class ServiceCollectionExtensions services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); services.AddSingleton(); // Provenance scope services (backport integration) @@ -105,6 +112,7 @@ public static class ServiceCollectionExtensions Action configureOptions) { services.Configure(configureOptions); + services.TryAddSingleton(TimeProvider.System); services.AddSingleton(); services.AddStartupMigrations( ConcelierDataSource.DefaultSchemaName, @@ -150,6 +158,10 @@ public static class ServiceCollectionExtensions services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); services.AddSingleton(); // Provenance scope services (backport integration) diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/SourceStateAdapter.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/SourceStateAdapter.cs index 92080741a..65af7f1e7 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/SourceStateAdapter.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/SourceStateAdapter.cs @@ -1,6 +1,7 @@ using Contracts = StellaOps.Concelier.Storage.Contracts; using LegacyContracts = StellaOps.Concelier.Storage; +using StellaOps.Concelier.Core.Sources; using StellaOps.Concelier.Documents; using StellaOps.Concelier.Persistence.Postgres.Models; using StellaOps.Concelier.Persistence.Postgres.Repositories; @@ -34,7 +35,7 @@ public sealed class PostgresSourceStateAdapter : LegacyContracts.ISourceStateRep { ArgumentException.ThrowIfNullOrEmpty(sourceName); - var source = await _sourceRepository.GetByKeyAsync(sourceName, cancellationToken).ConfigureAwait(false); + var source = await FindSourceAsync(sourceName, cancellationToken).ConfigureAwait(false); if (source is null) { return null; @@ -152,19 +153,37 @@ public sealed class PostgresSourceStateAdapter : LegacyContracts.ISourceStateRep private async Task EnsureSourceAsync(string sourceName, CancellationToken cancellationToken) { - var existing = await _sourceRepository.GetByKeyAsync(sourceName, cancellationToken).ConfigureAwait(false); + var normalizedSourceName = SourceKeyAliases.Normalize(sourceName); + var existing = await FindSourceAsync(normalizedSourceName, cancellationToken).ConfigureAwait(false); if (existing is not null) { - return existing; + return string.Equals(existing.Key, normalizedSourceName, StringComparison.OrdinalIgnoreCase) + ? existing + : await _sourceRepository.UpsertAsync( + new SourceEntity + { + Id = Guid.NewGuid(), + Key = normalizedSourceName, + Name = normalizedSourceName, + SourceType = normalizedSourceName, + Url = existing.Url, + Priority = existing.Priority, + Enabled = existing.Enabled, + Config = existing.Config, + Metadata = existing.Metadata, + CreatedAt = existing.CreatedAt, + UpdatedAt = _timeProvider.GetUtcNow() + }, + cancellationToken).ConfigureAwait(false); } var now = _timeProvider.GetUtcNow(); return await _sourceRepository.UpsertAsync(new SourceEntity { Id = Guid.NewGuid(), - Key = sourceName, - Name = sourceName, - SourceType = sourceName, + Key = normalizedSourceName, + Name = normalizedSourceName, + SourceType = normalizedSourceName, Url = null, Priority = 0, Enabled = true, @@ -175,6 +194,20 @@ public sealed class PostgresSourceStateAdapter : LegacyContracts.ISourceStateRep }, cancellationToken).ConfigureAwait(false); } + private async Task FindSourceAsync(string sourceName, CancellationToken cancellationToken) + { + foreach (var candidate in SourceKeyAliases.GetEquivalentKeys(sourceName)) + { + var source = await _sourceRepository.GetByKeyAsync(candidate, cancellationToken).ConfigureAwait(false); + if (source is not null) + { + return source; + } + } + + return null; + } + private static DateTimeOffset SafeAdd(DateTimeOffset value, TimeSpan delta) { try diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/TASKS.md b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/TASKS.md index 669e65599..3ca8ca5fb 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/TASKS.md +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/TASKS.md @@ -13,4 +13,7 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | TASK-015-007d | DONE | Added license indexes and repository queries for license inventory. | | BE8-07 | DONE | Added migration `005_add_advisory_source_signature_projection.sql` and advisory-source signature stats projection fields for UI detail diagnostics. | | REALPLAN-007-C | DONE | 2026-04-15: Added durable `PostgresLeaseStore` plus migration `007_add_job_leases.sql`; `AddConcelierPostgresStorage` now owns live Concelier lease coordination. | -| REALPLAN-007-F | DOING | 2026-04-19: Adding durable advisory-observation and affected-symbol persistence so live Concelier ingest can back `/v1/signals/symbols/*` with PostgreSQL instead of unsupported runtime fallbacks. | +| REALPLAN-007-F | DONE | 2026-04-19: Added durable advisory-observation and affected-symbol persistence so live Concelier ingest can back `/v1/signals/symbols/*` with PostgreSQL instead of unsupported runtime fallbacks. Verified with targeted registration and repository tests. | +| REALPLAN-007-G | DONE | 2026-04-19: Added migration `009_add_job_runs_and_orchestrator_runtime.sql`, `PostgresJobStore`, and `PostgresOrchestratorRegistryStore` so live `/jobs` and `/internal/orch/*` runtime state persists in PostgreSQL. Verified with targeted DI and repository coverage. | +| FE-ADVISORY-003-DATA | DONE | 2026-04-21: Added migration `010_add_advisory_source_content_counts.sql` plus projection/repository fields for `sourceDocumentCount`, `canonicalAdvisoryCount`, `cveCount`, and `vexDocumentCount` on live advisory-source read models. | +| CONN-ALIGN-003 | DOING | 2026-04-21: Tightening `PostgresJobStore.GetActiveRunsAsync()` so stale `Pending`/`Running` rows without a live lease stop blocking manual source syncs and `/jobs/active`; adding regression coverage for lease-backed active-run selection. | diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Attestation/Extensions/ServiceCollectionExtensions.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Attestation/Extensions/ServiceCollectionExtensions.cs index 895c078ae..f1414bdfa 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Attestation/Extensions/ServiceCollectionExtensions.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Attestation/Extensions/ServiceCollectionExtensions.cs @@ -1,4 +1,7 @@ using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Cryptography; using StellaOps.Excititor.Attestation.Dsse; using StellaOps.Excititor.Attestation.Evidence; using StellaOps.Excititor.Attestation.Transparency; @@ -14,7 +17,13 @@ public static class VexAttestationServiceCollectionExtensions { services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(provider => + new VexAttestationVerifier( + provider.GetRequiredService>(), + provider.GetService(), + provider.GetRequiredService>(), + provider.GetRequiredService(), + provider.GetService())); services.AddSingleton(); services.AddSingleton(); return services; diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Cisco.CSAF/CiscoCsafConnector.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Cisco.CSAF/CiscoCsafConnector.cs index 19be953a7..b170fead6 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Cisco.CSAF/CiscoCsafConnector.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Cisco.CSAF/CiscoCsafConnector.cs @@ -18,6 +18,9 @@ namespace StellaOps.Excititor.Connectors.Cisco.CSAF; public sealed class CiscoCsafConnector : VexConnectorBase { + private const string KnownFallbackPathsTokenKey = "cisco.csaf.indexPaths"; + private const string DeniedTimestampedPathsTokenKey = "cisco.csaf.deniedTimestampedPaths"; + private static readonly VexConnectorDescriptor DescriptorInstance = new( id: "excititor:cisco", kind: VexProviderKind.Vendor, @@ -85,23 +88,86 @@ public sealed class CiscoCsafConnector : VexConnectorBase var knownDigests = state?.DocumentDigests ?? ImmutableArray.Empty; var digestSet = new HashSet(knownDigests, StringComparer.OrdinalIgnoreCase); var digestList = new List(knownDigests); - var since = context.Since ?? state?.LastUpdated ?? DateTimeOffset.MinValue; - var latestTimestamp = state?.LastUpdated ?? since; + var resumeTokens = state?.ResumeTokens ?? context.ResumeTokens; + var knownFallbackPaths = LoadKnownFallbackPaths(resumeTokens); + var deniedTimestampedPaths = LoadDeniedTimestampedPaths(resumeTokens); + var persistedSince = knownDigests.Length > 0 ? state?.LastUpdated : null; + var since = context.Since ?? persistedSince ?? DateTimeOffset.MinValue; + var isBootstrapCursor = since == DateTimeOffset.MinValue; + var latestTimestamp = persistedSince ?? since; var stateChanged = false; + var bootstrapIncomplete = false; + var requestedDocumentCount = 0; + + LogConnectorEvent(LogLevel.Information, "fetch", "Cisco CSAF fetch starting.", new Dictionary + { + ["since"] = since == DateTimeOffset.MinValue ? null : since.ToString("O", CultureInfo.InvariantCulture), + ["knownDigestCount"] = knownDigests.Length, + ["knownFallbackPathCount"] = knownFallbackPaths.Count, + ["deniedTimestampedPathCount"] = deniedTimestampedPaths.Count, + ["maxDocumentsPerFetch"] = _options.MaxDocumentsPerFetch, + ["requestDelayMs"] = (int)_options.RequestDelay.TotalMilliseconds, + ["resumeTokenCount"] = context.ResumeTokens.Count, + }); var client = _httpClientFactory.CreateClient(CiscoConnectorOptions.HttpClientName); foreach (var directory in _providerMetadata.Provider.BaseUris) { await foreach (var advisory in EnumerateCatalogAsync(client, directory, cancellationToken).ConfigureAwait(false)) { + var hasPublishedTimestamp = advisory.LastModified is not null || advisory.Published is not null; + var useFallbackPathCheckpoint = !hasPublishedTimestamp || isBootstrapCursor; var published = advisory.LastModified ?? advisory.Published ?? DateTimeOffset.MinValue; - if (published <= since) + if (hasPublishedTimestamp && published <= since) { continue; } + if (useFallbackPathCheckpoint && knownFallbackPaths.Contains(advisory.Id)) + { + continue; + } + + if (hasPublishedTimestamp + && deniedTimestampedPaths.TryGetValue(advisory.Id, out var deniedPublished) + && published <= deniedPublished) + { + continue; + } + + if (requestedDocumentCount >= _options.MaxDocumentsPerFetch) + { + bootstrapIncomplete = isBootstrapCursor; + continue; + } + + requestedDocumentCount++; using var contentResponse = await client.GetAsync(advisory.DocumentUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - contentResponse.EnsureSuccessStatusCode(); + if (!contentResponse.IsSuccessStatusCode) + { + var statusCode = (int)contentResponse.StatusCode; + if (statusCode == 403 || statusCode == 404) + { + if (hasPublishedTimestamp) + { + deniedTimestampedPaths[advisory.Id] = published; + stateChanged = true; + } + else if (useFallbackPathCheckpoint && knownFallbackPaths.Add(advisory.Id)) + { + stateChanged = true; + } + + Logger.LogWarning( + "Cisco advisory document {DocumentUri} returned {StatusCode}; checkpointing candidate and continuing.", + advisory.DocumentUri, + contentResponse.StatusCode); + continue; + } + + contentResponse.EnsureSuccessStatusCode(); + } + var payload = await contentResponse.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); var metadata = BuildMetadata(builder => @@ -122,20 +188,41 @@ public sealed class CiscoCsafConnector : VexConnectorBase payload, metadata); + if (hasPublishedTimestamp && deniedTimestampedPaths.Remove(advisory.Id)) + { + stateChanged = true; + } + + var trackFallbackPath = useFallbackPathCheckpoint && knownFallbackPaths.Add(advisory.Id); if (!digestSet.Add(rawDocument.Digest)) { + if (trackFallbackPath) + { + stateChanged = true; + } + continue; } await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false); digestList.Add(rawDocument.Digest); stateChanged = true; + if (trackFallbackPath) + { + stateChanged = true; + } + if (published > latestTimestamp) { latestTimestamp = published; } yield return rawDocument; + + if (_options.RequestDelay > TimeSpan.Zero) + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } } } @@ -150,10 +237,20 @@ public sealed class CiscoCsafConnector : VexConnectorBase 0, null, null); + var checkpointTimestamp = bootstrapIncomplete + ? baseState.LastUpdated ?? DateTimeOffset.MinValue + : latestTimestamp; + if (!bootstrapIncomplete && checkpointTimestamp == DateTimeOffset.MinValue && knownFallbackPaths.Count > 0) + { + checkpointTimestamp = TimeProvider.GetUtcNow(); + } var newState = baseState with { - LastUpdated = latestTimestamp == DateTimeOffset.MinValue ? state?.LastUpdated : latestTimestamp, + LastUpdated = checkpointTimestamp == DateTimeOffset.MinValue ? state?.LastUpdated : checkpointTimestamp, DocumentDigests = digestList.ToImmutableArray(), + ResumeTokens = StoreDeniedTimestampedPaths( + StoreKnownFallbackPaths(baseState.ResumeTokens, knownFallbackPaths), + deniedTimestampedPaths), }; await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false); } @@ -164,6 +261,46 @@ public sealed class CiscoCsafConnector : VexConnectorBase private async IAsyncEnumerable EnumerateCatalogAsync(HttpClient client, Uri directory, [EnumeratorCancellation] CancellationToken cancellationToken) { + Exception? catalogFailure = null; + IReadOnlyList catalogEntries = Array.Empty(); + try + { + catalogEntries = await FetchPagedCatalogEntriesAsync(client, directory, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + catalogFailure = ex; + Logger.LogWarning( + ex, + "Cisco advisory index under {Directory} is unavailable; falling back to directory indexes.", + directory); + } + + if (catalogEntries.Count > 0) + { + foreach (var advisory in catalogEntries) + { + yield return advisory; + } + + yield break; + } + + var fallbackEntries = await FetchDirectoryEntriesAsync(client, directory, cancellationToken).ConfigureAwait(false); + if (fallbackEntries.Count == 0 && catalogFailure is not null) + { + throw new InvalidOperationException($"Cisco advisory root {directory} did not expose a usable catalog.", catalogFailure); + } + + foreach (var advisory in fallbackEntries) + { + yield return advisory; + } + } + + private async Task> FetchPagedCatalogEntriesAsync(HttpClient client, Uri directory, CancellationToken cancellationToken) + { + var builder = new List(); var nextUri = BuildIndexUri(directory, null); while (nextUri is not null) { @@ -173,7 +310,7 @@ public sealed class CiscoCsafConnector : VexConnectorBase var page = JsonSerializer.Deserialize(json, _serializerOptions); if (page?.Advisories is null) { - yield break; + break; } foreach (var advisory in page.Advisories) @@ -193,17 +330,172 @@ public sealed class CiscoCsafConnector : VexConnectorBase documentUri = new Uri(directory, documentUri); } - yield return new CiscoAdvisoryEntry( + builder.Add(new CiscoAdvisoryEntry( advisory.Id ?? documentUri.Segments.LastOrDefault()?.Trim('/') ?? documentUri.ToString(), documentUri, advisory.Revision, advisory.Published, advisory.LastModified, - advisory.Sha256); + advisory.Sha256)); } nextUri = ResolveNextUri(directory, page.Next); } + + return builder; + } + + private async Task> FetchDirectoryEntriesAsync(HttpClient client, Uri advisoryRoot, CancellationToken cancellationToken) + { + var changeEntries = await TryFetchChangeEntriesAsync(client, advisoryRoot, cancellationToken).ConfigureAwait(false); + if (changeEntries.Count > 0) + { + Logger.LogInformation( + "Cisco advisory change index under {AdvisoryRoot} yielded {EntryCount} candidate advisory document(s).", + advisoryRoot, + changeEntries.Count); + return changeEntries; + } + + var indexEntries = await TryFetchIndexEntriesAsync(client, advisoryRoot, cancellationToken).ConfigureAwait(false); + if (indexEntries.Count > 0) + { + Logger.LogInformation( + "Cisco advisory path index under {AdvisoryRoot} yielded {EntryCount} candidate advisory document(s).", + advisoryRoot, + indexEntries.Count); + return indexEntries; + } + + Logger.LogWarning("Cisco advisory directory fallback under {AdvisoryRoot} yielded zero candidate advisory documents.", advisoryRoot); + return Array.Empty(); + } + + private async Task> TryFetchChangeEntriesAsync(HttpClient client, Uri advisoryRoot, CancellationToken cancellationToken) + { + var changesUri = new Uri(advisoryRoot, "changes.csv"); + try + { + using var response = await client.GetAsync(changesUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + var content = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var builder = new List(); + foreach (var line in content.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + if (TryParseChangeEntry(line, advisoryRoot, out var entry)) + { + builder.Add(entry); + } + } + + return builder + .OrderByDescending(static entry => entry.LastModified ?? DateTimeOffset.MinValue) + .ThenByDescending(static entry => entry.DocumentUri.ToString(), StringComparer.Ordinal) + .ToList(); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Logger.LogWarning(ex, "Failed to fetch Cisco advisory change index from {ChangesUri}.", changesUri); + return Array.Empty(); + } + } + + private async Task> TryFetchIndexEntriesAsync(HttpClient client, Uri advisoryRoot, CancellationToken cancellationToken) + { + var indexUri = new Uri(advisoryRoot, "index.txt"); + try + { + using var response = await client.GetAsync(indexUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + var content = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var builder = new List(); + foreach (var line in content.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + if (TryParseIndexEntry(line, advisoryRoot, out var entry)) + { + builder.Add(entry); + } + } + + return builder + .OrderByDescending(static entry => entry.DocumentUri.ToString(), StringComparer.Ordinal) + .ToList(); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Logger.LogWarning(ex, "Failed to fetch Cisco advisory path index from {IndexUri}.", indexUri); + return Array.Empty(); + } + } + + private static bool TryParseChangeEntry(string line, Uri advisoryRoot, out CiscoAdvisoryEntry entry) + { + entry = default!; + if (string.IsNullOrWhiteSpace(line)) + { + return false; + } + + var trimmed = line.Trim(); + var separatorIndex = trimmed.IndexOf(',', StringComparison.Ordinal); + if (separatorIndex <= 0 || separatorIndex >= trimmed.Length - 1) + { + return false; + } + + var relativePath = trimmed[..separatorIndex].Trim().Trim('"'); + var updatedText = trimmed[(separatorIndex + 1)..].Trim().Trim('"'); + if (!TryCreateDocumentUri(advisoryRoot, relativePath, out var documentUri)) + { + return false; + } + + entry = new CiscoAdvisoryEntry( + relativePath, + documentUri, + Revision: null, + Published: null, + LastModified: DateTimeOffset.TryParse(updatedText, out var updated) ? updated : null, + Sha256: null); + return true; + } + + private static bool TryParseIndexEntry(string line, Uri advisoryRoot, out CiscoAdvisoryEntry entry) + { + entry = default!; + var relativePath = line.Trim(); + if (!TryCreateDocumentUri(advisoryRoot, relativePath, out var documentUri)) + { + return false; + } + + entry = new CiscoAdvisoryEntry( + relativePath, + documentUri, + Revision: null, + Published: null, + LastModified: null, + Sha256: null); + return true; + } + + private static bool TryCreateDocumentUri(Uri advisoryRoot, string relativePath, out Uri documentUri) + { + documentUri = default!; + if (string.IsNullOrWhiteSpace(relativePath) || relativePath.EndsWith("/", StringComparison.Ordinal)) + { + return false; + } + + if (!Uri.TryCreate(advisoryRoot, relativePath, out var parsedUri) || parsedUri is null) + { + return false; + } + + documentUri = parsedUri; + return documentUri.IsAbsoluteUri; } private static Uri BuildIndexUri(Uri directory, string? relative) @@ -243,6 +535,67 @@ public sealed class CiscoCsafConnector : VexConnectorBase return BuildIndexUri(directory, next); } + private static HashSet LoadKnownFallbackPaths(ImmutableDictionary resumeTokens) + { + var paths = new HashSet(StringComparer.Ordinal); + if (!resumeTokens.TryGetValue(KnownFallbackPathsTokenKey, out var serializedPaths) || string.IsNullOrWhiteSpace(serializedPaths)) + { + return paths; + } + + foreach (var path in serializedPaths.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + paths.Add(path); + } + + return paths; + } + + private static ImmutableDictionary StoreKnownFallbackPaths( + ImmutableDictionary resumeTokens, + HashSet knownFallbackPaths) + { + if (knownFallbackPaths.Count == 0) + { + return resumeTokens.Remove(KnownFallbackPathsTokenKey); + } + + var serializedPaths = string.Join( + "\n", + knownFallbackPaths.OrderBy(static path => path, StringComparer.Ordinal)); + + return resumeTokens.SetItem(KnownFallbackPathsTokenKey, serializedPaths); + } + + private static Dictionary LoadDeniedTimestampedPaths(ImmutableDictionary resumeTokens) + { + if (!resumeTokens.TryGetValue(DeniedTimestampedPathsTokenKey, out var serializedPaths) || string.IsNullOrWhiteSpace(serializedPaths)) + { + return new Dictionary(StringComparer.Ordinal); + } + + var parsed = JsonSerializer.Deserialize>(serializedPaths); + return parsed is null + ? new Dictionary(StringComparer.Ordinal) + : new Dictionary(parsed, StringComparer.Ordinal); + } + + private static ImmutableDictionary StoreDeniedTimestampedPaths( + ImmutableDictionary resumeTokens, + Dictionary deniedTimestampedPaths) + { + if (deniedTimestampedPaths.Count == 0) + { + return resumeTokens.Remove(DeniedTimestampedPathsTokenKey); + } + + var serializedPaths = JsonSerializer.Serialize( + deniedTimestampedPaths.OrderBy(static item => item.Key, StringComparer.Ordinal) + .ToDictionary(static item => item.Key, static item => item.Value, StringComparer.Ordinal)); + + return resumeTokens.SetItem(DeniedTimestampedPathsTokenKey, serializedPaths); + } + private void AddProvenanceMetadata(VexConnectorMetadataBuilder builder) { if (_providerMetadata is null) diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptions.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptions.cs index 4531ba659..d7b54b22b 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptions.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptions.cs @@ -4,13 +4,16 @@ namespace StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration; public sealed class CiscoConnectorOptions : IValidatableObject { + public const string DefaultMetadataUri = "https://www.cisco.com/.well-known/csaf/provider-metadata.json"; + public const string DefaultAdvisoriesDirectoryUri = "https://www.cisco.com/.well-known/csaf/"; + public const string HttpClientName = "cisco-csaf"; /// /// Endpoint for Cisco CSAF provider metadata discovery. /// [Required] - public string MetadataUri { get; set; } = "https://api.security.cisco.com/.well-known/csaf/provider-metadata.json"; + public string MetadataUri { get; set; } = DefaultMetadataUri; /// /// Optional bearer token used when Cisco endpoints require authentication. @@ -34,6 +37,31 @@ public sealed class CiscoConnectorOptions : IValidatableObject public string? OfflineSnapshotPath { get; set; } + /// + /// When true, a built-in Cisco provider snapshot is used if metadata discovery is unavailable. + /// + public bool AllowBuiltInSnapshotFallback { get; set; } = true; + + /// + /// Advisory directory used by the built-in fallback provider. + /// + public string BuiltInAdvisoriesDirectoryUri { get; set; } = DefaultAdvisoriesDirectoryUri; + + /// + /// Provider metadata document used by the built-in fallback provider. + /// + public string BuiltInMetadataUri { get; set; } = DefaultMetadataUri; + + /// + /// Maximum number of Cisco advisory documents requested during one fetch pass. + /// + public int MaxDocumentsPerFetch { get; set; } = 20; + + /// + /// Delay between Cisco advisory document requests during live bootstrap. + /// + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromSeconds(1); + public IEnumerable Validate(ValidationContext validationContext) { if (string.IsNullOrWhiteSpace(MetadataUri)) @@ -50,6 +78,26 @@ public sealed class CiscoConnectorOptions : IValidatableObject yield return new ValidationResult("MetadataCacheDuration must be greater than zero.", new[] { nameof(MetadataCacheDuration) }); } + if (!Uri.TryCreate(BuiltInAdvisoriesDirectoryUri, UriKind.Absolute, out _)) + { + yield return new ValidationResult("BuiltInAdvisoriesDirectoryUri must be an absolute URI.", new[] { nameof(BuiltInAdvisoriesDirectoryUri) }); + } + + if (!Uri.TryCreate(BuiltInMetadataUri, UriKind.Absolute, out _)) + { + yield return new ValidationResult("BuiltInMetadataUri must be an absolute URI.", new[] { nameof(BuiltInMetadataUri) }); + } + + if (MaxDocumentsPerFetch <= 0 || MaxDocumentsPerFetch > 200) + { + yield return new ValidationResult("MaxDocumentsPerFetch must be between 1 and 200.", new[] { nameof(MaxDocumentsPerFetch) }); + } + + if (RequestDelay < TimeSpan.Zero || RequestDelay > TimeSpan.FromMinutes(5)) + { + yield return new ValidationResult("RequestDelay must be between 0 and 5 minutes.", new[] { nameof(RequestDelay) }); + } + if (PersistOfflineSnapshot && string.IsNullOrWhiteSpace(OfflineSnapshotPath)) { yield return new ValidationResult("OfflineSnapshotPath must be provided when PersistOfflineSnapshot is enabled.", new[] { nameof(OfflineSnapshotPath) }); diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Cisco.CSAF/DependencyInjection/CiscoConnectorServiceCollectionExtensions.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Cisco.CSAF/DependencyInjection/CiscoConnectorServiceCollectionExtensions.cs index 62148404d..9b6b93eb7 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Cisco.CSAF/DependencyInjection/CiscoConnectorServiceCollectionExtensions.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Cisco.CSAF/DependencyInjection/CiscoConnectorServiceCollectionExtensions.cs @@ -9,12 +9,15 @@ using StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata; using StellaOps.Excititor.Core; using System.ComponentModel.DataAnnotations; using System.IO.Abstractions; +using System.Net.Http; using System.Net.Http.Headers; namespace StellaOps.Excititor.Connectors.Cisco.CSAF.DependencyInjection; public static class CiscoConnectorServiceCollectionExtensions { + private const string CiscoJsonCatalogUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"; + public static IServiceCollection AddCiscoCsafConnector(this IServiceCollection services, Action? configure = null) { ArgumentNullException.ThrowIfNull(services); @@ -38,7 +41,17 @@ public static class CiscoConnectorServiceCollectionExtensions { var options = provider.GetRequiredService>().Value; client.Timeout = TimeSpan.FromSeconds(30); + client.DefaultRequestVersion = new Version(1, 1); + client.DefaultVersionPolicy = HttpVersionPolicy.RequestVersionExact; + client.DefaultRequestHeaders.Accept.Clear(); + client.DefaultRequestHeaders.AcceptLanguage.Clear(); + client.DefaultRequestHeaders.UserAgent.Clear(); + client.DefaultRequestHeaders.Accept.ParseAdd("application/csaf+json"); client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); + client.DefaultRequestHeaders.Accept.ParseAdd("text/plain"); + client.DefaultRequestHeaders.Accept.ParseAdd("*/*"); + client.DefaultRequestHeaders.AcceptLanguage.ParseAdd("en-US,en;q=0.9"); + client.DefaultRequestHeaders.UserAgent.ParseAdd(CiscoJsonCatalogUserAgent); if (!string.IsNullOrWhiteSpace(options.ApiToken)) { client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", options.ApiToken); diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Cisco.CSAF/Metadata/CiscoProviderMetadataLoader.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Cisco.CSAF/Metadata/CiscoProviderMetadataLoader.cs index a0500239d..edb3bd5da 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Cisco.CSAF/Metadata/CiscoProviderMetadataLoader.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Cisco.CSAF/Metadata/CiscoProviderMetadataLoader.cs @@ -90,6 +90,16 @@ public sealed class CiscoProviderMetadataLoader return new CiscoProviderMetadataResult(entry.Provider, entry.FetchedAt, true, false); } + if (_options.AllowBuiltInSnapshotFallback) + { + var fallback = CreateBuiltInFallbackEntry(); + StoreCache(fallback); + _logger.LogWarning( + "Falling back to built-in Cisco CSAF provider metadata for {Directory} because live discovery and offline snapshots were unavailable.", + _options.BuiltInAdvisoriesDirectoryUri); + return new CiscoProviderMetadataResult(fallback.Provider, fallback.FetchedAt, true, false); + } + throw new InvalidOperationException("Unable to load Cisco CSAF provider metadata from network or offline snapshot."); } finally @@ -98,6 +108,27 @@ public sealed class CiscoProviderMetadataLoader } } + private CacheEntry CreateBuiltInFallbackEntry() + { + var provider = new VexProvider( + id: "excititor:cisco", + displayName: "Cisco PSIRT", + kind: VexProviderKind.Vendor, + baseUris: [CreateUri(_options.BuiltInAdvisoriesDirectoryUri, nameof(CiscoConnectorOptions.BuiltInAdvisoriesDirectoryUri))], + discovery: new VexProviderDiscovery( + CreateUri(_options.BuiltInMetadataUri, nameof(CiscoConnectorOptions.BuiltInMetadataUri)), + null), + trust: BuildDefaultTrust()); + + var now = _timeProvider.GetUtcNow(); + return new CacheEntry( + provider, + now, + now + _options.MetadataCacheDuration, + ETag: null, + FromOffline: true); + } + private async Task TryFetchFromNetworkAsync(CacheEntry? previous, CancellationToken cancellationToken) { try @@ -201,37 +232,105 @@ public sealed class CiscoProviderMetadataLoader throw new InvalidOperationException("Failed to parse Cisco provider metadata.", ex); } - if (document?.Metadata?.Publisher?.ContactDetails is null || string.IsNullOrWhiteSpace(document.Metadata.Publisher.ContactDetails.Id)) + if (document is null) { - throw new InvalidOperationException("Cisco provider metadata did not include a publisher identifier."); + throw new InvalidOperationException("Cisco provider metadata payload was null after parsing."); } - var discovery = new VexProviderDiscovery(document.Discovery?.WellKnown, document.Discovery?.RolIe); - var trust = document.Trust is null - ? VexProviderTrust.Default - : new VexProviderTrust( - document.Trust.Weight ?? 1.0, - document.Trust.Cosign is null ? null : new VexCosignTrust(document.Trust.Cosign.Issuer ?? string.Empty, document.Trust.Cosign.IdentityPattern ?? string.Empty), - document.Trust.PgpFingerprints ?? Enumerable.Empty()); + var publisher = document.Metadata?.Publisher ?? document.Publisher; + if (publisher is null || string.IsNullOrWhiteSpace(publisher.Name)) + { + throw new InvalidOperationException("Cisco provider metadata did not include a publisher name."); + } - var directories = document.Distributions?.Directories is null - ? Enumerable.Empty() - : document.Distributions.Directories - .Where(static s => !string.IsNullOrWhiteSpace(s)) - .Select(static s => Uri.TryCreate(s, UriKind.Absolute, out var uri) ? uri : null) - .Where(static uri => uri is not null)! - .Select(static uri => uri!); + var discovery = new VexProviderDiscovery( + document.Discovery?.WellKnown ?? CreateUri(_options.MetadataUri, nameof(CiscoConnectorOptions.MetadataUri)), + document.Discovery?.RolIe); + var trust = BuildTrust(document); + var directories = ParseDirectories(document.Distributions).ToArray(); + if (directories.Length == 0) + { + throw new InvalidOperationException("Cisco provider metadata did not include any valid distribution directories."); + } return new VexProvider( - id: document.Metadata.Publisher.ContactDetails.Id, - displayName: document.Metadata.Publisher.Name ?? document.Metadata.Publisher.ContactDetails.Id, - kind: document.Metadata.Publisher.Category?.Equals("vendor", StringComparison.OrdinalIgnoreCase) == true ? VexProviderKind.Vendor : VexProviderKind.Hub, + id: "excititor:cisco", + displayName: publisher.Name, + kind: publisher.Category?.Equals("vendor", StringComparison.OrdinalIgnoreCase) == true ? VexProviderKind.Vendor : VexProviderKind.Hub, baseUris: directories, discovery: discovery, trust: trust, enabled: true); } + private static IEnumerable ParseDirectories(JsonElement distributions) + { + if (distributions.ValueKind == JsonValueKind.Array) + { + foreach (var entry in distributions.EnumerateArray()) + { + if (entry.ValueKind == JsonValueKind.Object && + entry.TryGetProperty("directory_url", out var directoryUrl) && + directoryUrl.ValueKind == JsonValueKind.String && + !string.IsNullOrWhiteSpace(directoryUrl.GetString())) + { + yield return CreateUri(directoryUrl.GetString()!, "distributions[].directory_url"); + } + } + + yield break; + } + + if (distributions.ValueKind == JsonValueKind.Object && + distributions.TryGetProperty("directories", out var directories) && + directories.ValueKind == JsonValueKind.Array) + { + foreach (var entry in directories.EnumerateArray()) + { + if (entry.ValueKind == JsonValueKind.String && !string.IsNullOrWhiteSpace(entry.GetString())) + { + yield return CreateUri(entry.GetString()!, "distributions.directories[]"); + } + } + } + } + + private static VexProviderTrust BuildTrust(ProviderMetadataDocument document) + { + var publicKeyFingerprints = document.PublicOpenPgpKeys? + .Select(static key => key.Fingerprint) + .Where(static fingerprint => !string.IsNullOrWhiteSpace(fingerprint)) + .Select(static fingerprint => fingerprint!) + .ToArray() ?? Array.Empty(); + + if (document.Trust is not null) + { + var configuredFingerprints = document.Trust.PgpFingerprints? + .Where(static fingerprint => !string.IsNullOrWhiteSpace(fingerprint)) + .ToArray() ?? Array.Empty(); + return new VexProviderTrust( + document.Trust.Weight ?? 1.0, + document.Trust.Cosign is null ? null : new VexCosignTrust(document.Trust.Cosign.Issuer ?? string.Empty, document.Trust.Cosign.IdentityPattern ?? string.Empty), + configuredFingerprints.Concat(publicKeyFingerprints).Distinct(StringComparer.OrdinalIgnoreCase)); + } + + if (publicKeyFingerprints.Length > 0) + { + return new VexProviderTrust( + VexProviderTrust.Default.Weight, + VexProviderTrust.Default.Cosign, + publicKeyFingerprints); + } + + return BuildDefaultTrust(); + } + + private static VexProviderTrust BuildDefaultTrust() + => new( + VexProviderTrust.Default.Weight, + VexProviderTrust.Default.Cosign, + VexProviderTrust.Default.PgpFingerprints); + private void StoreCache(CacheEntry entry) { var options = new MemoryCacheEntryOptions @@ -241,6 +340,16 @@ public sealed class CiscoProviderMetadataLoader _memoryCache.Set(CacheKey, entry, options); } + private static Uri CreateUri(string value, string propertyName) + { + if (!Uri.TryCreate(value, UriKind.Absolute, out var uri) || uri.Scheme is not ("http" or "https")) + { + throw new InvalidOperationException($"Cisco connector field '{propertyName}' must be an absolute HTTP(S) URI."); + } + + return uri; + } + private sealed record CacheEntry( VexProvider Provider, DateTimeOffset FetchedAt, @@ -263,7 +372,10 @@ public sealed record CiscoProviderMetadataResult( internal sealed class ProviderMetadataDocument { [System.Text.Json.Serialization.JsonPropertyName("metadata")] - public ProviderMetadataMetadata Metadata { get; set; } = new(); + public ProviderMetadataMetadata? Metadata { get; set; } + + [System.Text.Json.Serialization.JsonPropertyName("publisher")] + public ProviderMetadataPublisher? Publisher { get; set; } [System.Text.Json.Serialization.JsonPropertyName("discovery")] public ProviderMetadataDiscovery? Discovery { get; set; } @@ -272,13 +384,16 @@ internal sealed class ProviderMetadataDocument public ProviderMetadataTrust? Trust { get; set; } [System.Text.Json.Serialization.JsonPropertyName("distributions")] - public ProviderMetadataDistributions? Distributions { get; set; } + public JsonElement Distributions { get; set; } + + [System.Text.Json.Serialization.JsonPropertyName("public_openpgp_keys")] + public ProviderMetadataPublicOpenPgpKey[]? PublicOpenPgpKeys { get; set; } } internal sealed class ProviderMetadataMetadata { [System.Text.Json.Serialization.JsonPropertyName("publisher")] - public ProviderMetadataPublisher Publisher { get; set; } = new(); + public ProviderMetadataPublisher? Publisher { get; set; } } internal sealed class ProviderMetadataPublisher @@ -290,13 +405,10 @@ internal sealed class ProviderMetadataPublisher public string? Category { get; set; } [System.Text.Json.Serialization.JsonPropertyName("contact_details")] - public ProviderMetadataPublisherContact ContactDetails { get; set; } = new(); -} + public JsonElement ContactDetails { get; set; } -internal sealed class ProviderMetadataPublisherContact -{ - [System.Text.Json.Serialization.JsonPropertyName("id")] - public string? Id { get; set; } + [System.Text.Json.Serialization.JsonPropertyName("namespace")] + public string? Namespace { get; set; } } internal sealed class ProviderMetadataDiscovery @@ -329,10 +441,10 @@ internal sealed class ProviderMetadataTrustCosign public string? IdentityPattern { get; set; } } -internal sealed class ProviderMetadataDistributions +internal sealed class ProviderMetadataPublicOpenPgpKey { - [System.Text.Json.Serialization.JsonPropertyName("directories")] - public string[]? Directories { get; set; } + [System.Text.Json.Serialization.JsonPropertyName("fingerprint")] + public string? Fingerprint { get; set; } } #endregion diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptions.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptions.cs index a0c0abd55..c49fd0adc 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptions.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptions.cs @@ -7,11 +7,12 @@ namespace StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration; public sealed class OracleConnectorOptions { public const string HttpClientName = "excititor.connector.oracle.catalog"; + public static readonly Uri DefaultCatalogUri = new("https://www.oracle.com/ocom/groups/public/@otn/documents/webcontent/rss-otn-sec.xml"); /// - /// Oracle CSAF catalog endpoint hosting advisory metadata. + /// Oracle advisory discovery endpoint. The default is Oracle's official security RSS feed. /// - public Uri CatalogUri { get; set; } = new("https://www.oracle.com/security-alerts/cpu/csaf/catalog.json"); + public Uri CatalogUri { get; set; } = DefaultCatalogUri; /// /// Optional CPU calendar endpoint providing upcoming release dates. diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Oracle.CSAF/DependencyInjection/OracleConnectorServiceCollectionExtensions.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Oracle.CSAF/DependencyInjection/OracleConnectorServiceCollectionExtensions.cs index 063187b46..a54429891 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Oracle.CSAF/DependencyInjection/OracleConnectorServiceCollectionExtensions.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Oracle.CSAF/DependencyInjection/OracleConnectorServiceCollectionExtensions.cs @@ -32,6 +32,10 @@ public static class OracleConnectorServiceCollectionExtensions client.Timeout = TimeSpan.FromSeconds(60); client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.Oracle.CSAF/1.0"); client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); + client.DefaultRequestHeaders.Accept.ParseAdd("application/rss+xml"); + client.DefaultRequestHeaders.Accept.ParseAdd("application/xml"); + client.DefaultRequestHeaders.Accept.ParseAdd("text/xml"); + client.DefaultRequestHeaders.Accept.ParseAdd("text/html"); }) .ConfigurePrimaryHttpMessageHandler(static () => new HttpClientHandler { diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Oracle.CSAF/Metadata/OracleCatalogLoader.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Oracle.CSAF/Metadata/OracleCatalogLoader.cs index ce5a2ca13..35859a3ab 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Oracle.CSAF/Metadata/OracleCatalogLoader.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Oracle.CSAF/Metadata/OracleCatalogLoader.cs @@ -5,12 +5,14 @@ using StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration; using System; using System.Collections.Generic; using System.Collections.Immutable; +using System.Globalization; using System.IO.Abstractions; using System.Net; using System.Net.Http; using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using System.Xml.Linq; namespace StellaOps.Excititor.Connectors.Oracle.CSAF.Metadata; @@ -174,6 +176,22 @@ public sealed class OracleCatalogLoader throw new InvalidOperationException("Oracle catalog payload was empty."); } + var trimmedPayload = catalogPayload.AsSpan().TrimStart(); + var metadata = trimmedPayload.Length > 0 && trimmedPayload[0] == '<' + ? ParseRssMetadata(catalogPayload) + : ParseJsonMetadata(catalogPayload); + + var schedule = metadata.CpuSchedule; + if (!string.IsNullOrWhiteSpace(calendarPayload)) + { + schedule = MergeSchedule(schedule, calendarPayload); + } + + return metadata with { CpuSchedule = schedule }; + } + + private OracleCatalogMetadata ParseJsonMetadata(string catalogPayload) + { using var document = JsonDocument.Parse(catalogPayload); var root = document.RootElement; @@ -181,15 +199,35 @@ public sealed class OracleCatalogLoader ? generated : _timeProvider.GetUtcNow(); - var entries = ParseEntries(root); - var schedule = ParseSchedule(root); + return new OracleCatalogMetadata(generatedAt, ParseEntries(root), ParseSchedule(root)); + } - if (!string.IsNullOrWhiteSpace(calendarPayload)) + private OracleCatalogMetadata ParseRssMetadata(string catalogPayload) + { + var document = XDocument.Parse(catalogPayload, LoadOptions.None); + var builder = ImmutableArray.CreateBuilder(); + + foreach (var item in document.Descendants("item")) { - schedule = MergeSchedule(schedule, calendarPayload); + if (TryParseRssEntry(item, out var entry)) + { + builder.Add(entry); + } } - return new OracleCatalogMetadata(generatedAt, entries, schedule); + var entries = builder + .ToImmutable() + .OrderBy(static entry => entry.PublishedAt == default ? DateTimeOffset.MinValue : entry.PublishedAt) + .ThenBy(static entry => entry.Id, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + + var generatedAt = entries.Length == 0 + ? _timeProvider.GetUtcNow() + : entries + .Select(static entry => entry.PublishedAt == default ? DateTimeOffset.MinValue : entry.PublishedAt) + .Max(); + + return new OracleCatalogMetadata(generatedAt, entries, ImmutableArray.Empty); } private ImmutableArray ParseEntries(JsonElement root) @@ -383,6 +421,109 @@ public sealed class OracleCatalogLoader private static string CreateCacheKey(OracleConnectorOptions options) => $"{CachePrefix}:{options.CatalogUri}:{options.CpuCalendarUri}"; + private static bool TryParseRssEntry(XElement item, out OracleCatalogEntry entry) + { + entry = default!; + + var title = item.Element("title")?.Value?.Trim(); + var linkText = item.Element("link")?.Value?.Trim(); + if (string.IsNullOrWhiteSpace(title) || + string.IsNullOrWhiteSpace(linkText) || + !Uri.TryCreate(linkText, UriKind.Absolute, out var advisoryUri)) + { + return false; + } + + var advisoryStem = GetAdvisoryStem(advisoryUri); + if (string.IsNullOrWhiteSpace(advisoryStem) || + advisoryStem.Contains("announce", StringComparison.OrdinalIgnoreCase) || + title.Contains("announce", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + if (!TryDeriveDocumentUri(advisoryStem, out var documentUri, out var entryId)) + { + return false; + } + + var publishedAt = TryParseDate(item.Element("pubDate")?.Value) ?? DateTimeOffset.MinValue; + var revision = publishedAt == DateTimeOffset.MinValue + ? null + : publishedAt.ToString("O", CultureInfo.InvariantCulture); + + entry = new OracleCatalogEntry( + entryId, + title, + documentUri, + publishedAt, + revision, + null, + null, + ImmutableArray.Empty); + + return true; + } + + private static string GetAdvisoryStem(Uri advisoryUri) + { + var segments = advisoryUri.Segments; + if (segments.Length == 0) + { + return string.Empty; + } + + var fileName = segments[^1].Trim('/'); + if (fileName.EndsWith(".html", StringComparison.OrdinalIgnoreCase)) + { + fileName = fileName[..^5]; + } + + return fileName.Trim(); + } + + private static bool TryDeriveDocumentUri(string advisoryStem, out Uri documentUri, out string entryId) + { + documentUri = default!; + entryId = string.Empty; + + string documentStem; + if (advisoryStem.StartsWith("cpu", StringComparison.OrdinalIgnoreCase)) + { + documentStem = advisoryStem; + entryId = advisoryStem; + } + else if (advisoryStem.StartsWith("alert-cve-", StringComparison.OrdinalIgnoreCase)) + { + documentStem = advisoryStem["alert-".Length..]; + entryId = documentStem; + } + else if (advisoryStem.StartsWith("cve-", StringComparison.OrdinalIgnoreCase)) + { + documentStem = advisoryStem; + entryId = advisoryStem; + } + else + { + return false; + } + + documentUri = new Uri($"https://www.oracle.com/docs/tech/security-alerts/{documentStem}csaf.json", UriKind.Absolute); + return true; + } + + private static DateTimeOffset? TryParseDate(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed) + ? parsed.ToUniversalTime() + : (DateTimeOffset?)null; + } + private sealed record CacheEntry(OracleCatalogMetadata Metadata, DateTimeOffset FetchedAt, DateTimeOffset CachedAt, TimeSpan MetadataCacheDuration, bool FromOfflineSnapshot) { public bool IsExpired(DateTimeOffset now) diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Oracle.CSAF/OracleCsafConnector.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Oracle.CSAF/OracleCsafConnector.cs index d4e9889f3..3cabe3aa0 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Oracle.CSAF/OracleCsafConnector.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Oracle.CSAF/OracleCsafConnector.cs @@ -84,11 +84,12 @@ public sealed class OracleCsafConnector : VexConnectorBase .ToImmutableArray(); var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false); - var since = ResolveSince(context.Since, state?.LastUpdated); var knownDigests = state?.DocumentDigests ?? ImmutableArray.Empty; + var persistedSince = knownDigests.Length > 0 ? state?.LastUpdated : null; + var since = ResolveSince(context.Since, persistedSince); var digestSet = new HashSet(knownDigests, StringComparer.OrdinalIgnoreCase); var digestList = new List(knownDigests); - var latestPublished = state?.LastUpdated ?? since ?? DateTimeOffset.MinValue; + var latestPublished = persistedSince ?? since ?? DateTimeOffset.MinValue; var stateChanged = false; var client = _httpClientFactory.CreateClient(OracleConnectorOptions.HttpClientName); @@ -287,6 +288,16 @@ public sealed class OracleCsafConnector : VexConnectorBase using var response = await client.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); if (!response.IsSuccessStatusCode) { + if (IsMissing(response.StatusCode)) + { + LogConnectorEvent(LogLevel.Warning, "fetch.skip.missing_document", "Oracle CSAF document URI is unavailable; entry skipped.", new Dictionary + { + ["status"] = (int)response.StatusCode, + ["uri"] = uri.ToString(), + }); + return null; + } + if (IsTransient(response.StatusCode) && attempt < maxAttempts) { LogConnectorEvent(LogLevel.Warning, "fetch.retry.status", "Oracle CSAF document request returned transient status; retrying.", new Dictionary @@ -330,6 +341,9 @@ public sealed class OracleCsafConnector : VexConnectorBase private static bool IsTransient(Exception exception) => exception is HttpRequestException or IOException or TaskCanceledException; + private static bool IsMissing(HttpStatusCode statusCode) + => statusCode is HttpStatusCode.NotFound or HttpStatusCode.Gone; + private static bool IsTransient(HttpStatusCode statusCode) { var status = (int)statusCode; diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.RedHat.CSAF/Configuration/RedHatConnectorOptions.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.RedHat.CSAF/Configuration/RedHatConnectorOptions.cs index 9e0e5f7e3..10c813b99 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.RedHat.CSAF/Configuration/RedHatConnectorOptions.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.RedHat.CSAF/Configuration/RedHatConnectorOptions.cs @@ -6,6 +6,8 @@ namespace StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration; public sealed class RedHatConnectorOptions { public static readonly Uri DefaultMetadataUri = new("https://access.redhat.com/.well-known/csaf/provider-metadata.json"); + public static readonly Uri DefaultAdvisoriesDirectoryUri = new("https://access.redhat.com/security/data/csaf/v2/advisories/"); + public static readonly Uri DefaultRolieFeedUri = new("https://access.redhat.com/security/data/csaf/v2/advisories/rolie/feed.atom"); /// /// HTTP client name registered for the connector. @@ -37,6 +39,11 @@ public sealed class RedHatConnectorOptions /// public bool PersistOfflineSnapshot { get; set; } = true; + /// + /// When true, a built-in Red Hat CSAF directory snapshot is used if network and offline discovery are unavailable. + /// + public bool AllowBuiltInSnapshotFallback { get; set; } = true; + /// /// Explicit trust weight override applied to the provider entry. /// diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.RedHat.CSAF/Metadata/RedHatProviderMetadataLoader.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.RedHat.CSAF/Metadata/RedHatProviderMetadataLoader.cs index dc05f1d04..44bf33aa1 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.RedHat.CSAF/Metadata/RedHatProviderMetadataLoader.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.RedHat.CSAF/Metadata/RedHatProviderMetadataLoader.cs @@ -88,6 +88,16 @@ public sealed class RedHatProviderMetadataLoader return new RedHatProviderMetadataResult(offlineEntry.Provider, offlineEntry.FetchedAt, false, true); } + if (_options.AllowBuiltInSnapshotFallback) + { + var fallbackEntry = CreateBuiltInFallbackEntry(); + StoreCache(fallbackEntry); + _logger.LogWarning( + "Falling back to built-in Red Hat CSAF provider metadata for {Directory} because live discovery and offline snapshots were unavailable.", + RedHatConnectorOptions.DefaultAdvisoriesDirectoryUri); + return new RedHatProviderMetadataResult(fallbackEntry.Provider, fallbackEntry.FetchedAt, false, true); + } + throw new InvalidOperationException("Unable to load Red Hat CSAF provider metadata from network or offline snapshot."); } finally @@ -96,6 +106,24 @@ public sealed class RedHatProviderMetadataLoader } } + private CacheEntry CreateBuiltInFallbackEntry() + { + var provider = new VexProvider( + id: "excititor:redhat", + displayName: "Red Hat Product Security", + kind: VexProviderKind.Distro, + baseUris: [RedHatConnectorOptions.DefaultAdvisoriesDirectoryUri], + discovery: new VexProviderDiscovery(_options.MetadataUri, RedHatConnectorOptions.DefaultRolieFeedUri), + trust: BuildTrust()); + + return new CacheEntry( + provider, + DateTimeOffset.UtcNow, + DateTimeOffset.UtcNow + _options.MetadataCacheDuration, + ETag: null, + FromOffline: true); + } + private void StoreCache(CacheEntry entry) { var cacheEntryOptions = new MemoryCacheEntryOptions diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.RedHat.CSAF/RedHatCsafConnector.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.RedHat.CSAF/RedHatCsafConnector.cs index eafb72588..fa6af9bea 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.RedHat.CSAF/RedHatCsafConnector.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.RedHat.CSAF/RedHatCsafConnector.cs @@ -19,17 +19,24 @@ namespace StellaOps.Excititor.Connectors.RedHat.CSAF; public sealed class RedHatCsafConnector : VexConnectorBase { + private static readonly VexConnectorDescriptor DescriptorInstance = new( + id: "excititor:redhat", + kind: VexProviderKind.Distro, + displayName: "Red Hat CSAF") + { + Tags = ImmutableArray.Create("redhat", "csaf", "rolie"), + }; + private readonly RedHatProviderMetadataLoader _metadataLoader; private readonly IHttpClientFactory _httpClientFactory; private readonly IVexConnectorStateRepository _stateRepository; public RedHatCsafConnector( - VexConnectorDescriptor descriptor, RedHatProviderMetadataLoader metadataLoader, IHttpClientFactory httpClientFactory, IVexConnectorStateRepository stateRepository, ILogger logger, TimeProvider timeProvider) - : base(descriptor, logger, timeProvider) + : base(DescriptorInstance, logger, timeProvider) { _metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader)); _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); @@ -47,10 +54,6 @@ public sealed class RedHatCsafConnector : VexConnectorBase ArgumentNullException.ThrowIfNull(context); var metadataResult = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false); - if (metadataResult.Provider.Discovery.RolIeService is null) - { - throw new InvalidOperationException("Red Hat provider metadata did not specify a ROLIE feed."); - } var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false); @@ -66,7 +69,7 @@ public sealed class RedHatCsafConnector : VexConnectorBase var latestUpdated = state?.LastUpdated ?? sinceTimestamp ?? DateTimeOffset.MinValue; var stateChanged = false; - foreach (var entry in await FetchRolieEntriesAsync(metadataResult.Provider.Discovery.RolIeService, cancellationToken).ConfigureAwait(false)) + foreach (var entry in await FetchEntriesAsync(metadataResult.Provider, cancellationToken).ConfigureAwait(false)) { if (sinceTimestamp is not null && entry.Updated is DateTimeOffset updated && updated <= sinceTimestamp) { @@ -121,6 +124,44 @@ public sealed class RedHatCsafConnector : VexConnectorBase } } + private async Task> FetchEntriesAsync(VexProvider provider, CancellationToken cancellationToken) + { + Exception? rolieFailure = null; + if (provider.Discovery.RolIeService is not null) + { + try + { + return await FetchRolieEntriesAsync(provider.Discovery.RolIeService, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + rolieFailure = ex; + Logger.LogWarning( + ex, + "Red Hat ROLIE feed {FeedUri} is unavailable; falling back to advisory directory indexes.", + provider.Discovery.RolIeService); + } + } + + var advisoryRoot = provider.BaseUris.FirstOrDefault(); + if (advisoryRoot is null) + { + throw rolieFailure is null + ? new InvalidOperationException("Red Hat provider metadata did not specify a usable advisory root.") + : new InvalidOperationException("Red Hat provider did not expose a usable advisory root after ROLIE discovery failed.", rolieFailure); + } + + var directoryEntries = await FetchDirectoryEntriesAsync(advisoryRoot, cancellationToken).ConfigureAwait(false); + if (directoryEntries.Count > 0) + { + return directoryEntries; + } + + throw rolieFailure is null + ? new InvalidOperationException($"Red Hat advisory root {advisoryRoot} did not expose any directory index entries.") + : new InvalidOperationException($"Red Hat advisory root {advisoryRoot} did not expose any directory index entries after ROLIE discovery failed.", rolieFailure); + } + public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) { // This connector relies on format-specific normalizers registered elsewhere. @@ -150,9 +191,152 @@ public sealed class RedHatCsafConnector : VexConnectorBase return entries; } + private async Task> FetchDirectoryEntriesAsync(Uri advisoryRoot, CancellationToken cancellationToken) + { + var changeEntries = await TryFetchChangeEntriesAsync(advisoryRoot, cancellationToken).ConfigureAwait(false); + if (changeEntries.Count > 0) + { + return changeEntries; + } + + var indexEntries = await TryFetchIndexEntriesAsync(advisoryRoot, cancellationToken).ConfigureAwait(false); + if (indexEntries.Count > 0) + { + return indexEntries; + } + + return Array.Empty(); + } + + private async Task> TryFetchChangeEntriesAsync(Uri advisoryRoot, CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(RedHatConnectorOptions.HttpClientName); + var changesUri = new Uri(advisoryRoot, "changes.csv"); + try + { + using var response = await client.GetAsync(changesUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + var content = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var builder = new List(); + foreach (var line in content.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + if (TryParseChangeEntry(line, advisoryRoot, out var entry)) + { + builder.Add(entry); + } + } + + return builder + .OrderBy(static entry => entry.Updated ?? DateTimeOffset.MinValue) + .ThenBy(static entry => entry.DocumentUri?.ToString(), StringComparer.Ordinal) + .ToList(); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Logger.LogWarning(ex, "Failed to fetch Red Hat advisory change index from {ChangesUri}.", changesUri); + return Array.Empty(); + } + } + + private async Task> TryFetchIndexEntriesAsync(Uri advisoryRoot, CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(RedHatConnectorOptions.HttpClientName); + var indexUri = new Uri(advisoryRoot, "index.txt"); + try + { + using var response = await client.GetAsync(indexUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + var content = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var builder = new List(); + foreach (var line in content.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + if (TryParseIndexEntry(line, advisoryRoot, out var entry)) + { + builder.Add(entry); + } + } + + return builder + .OrderBy(static entry => entry.DocumentUri?.ToString(), StringComparer.Ordinal) + .ToList(); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Logger.LogWarning(ex, "Failed to fetch Red Hat advisory path index from {IndexUri}.", indexUri); + return Array.Empty(); + } + } + private static DateTimeOffset? ParseUpdated(string? value) => DateTimeOffset.TryParse(value, out var parsed) ? parsed : null; + private static bool TryParseChangeEntry(string line, Uri advisoryRoot, out RolieEntry entry) + { + entry = default!; + if (string.IsNullOrWhiteSpace(line)) + { + return false; + } + + var trimmed = line.Trim(); + if (trimmed.Length < 5 || trimmed[0] != '"') + { + return false; + } + + var separatorIndex = trimmed.IndexOf("\",\"", StringComparison.Ordinal); + if (separatorIndex <= 1) + { + return false; + } + + var relativePath = trimmed[1..separatorIndex]; + var updatedText = trimmed[(separatorIndex + 3)..].TrimEnd('"'); + if (!TryCreateDocumentUri(advisoryRoot, relativePath, out var documentUri)) + { + return false; + } + + entry = new RolieEntry( + Id: relativePath, + Updated: ParseUpdated(updatedText), + DocumentUri: documentUri); + return true; + } + + private static bool TryParseIndexEntry(string line, Uri advisoryRoot, out RolieEntry entry) + { + entry = default!; + var relativePath = line.Trim(); + if (!TryCreateDocumentUri(advisoryRoot, relativePath, out var documentUri)) + { + return false; + } + + entry = new RolieEntry( + Id: relativePath, + Updated: null, + DocumentUri: documentUri); + return true; + } + + private static bool TryCreateDocumentUri(Uri advisoryRoot, string relativePath, out Uri? documentUri) + { + documentUri = null; + if (string.IsNullOrWhiteSpace(relativePath) + || !relativePath.EndsWith(".json", StringComparison.OrdinalIgnoreCase) + || relativePath.EndsWith(".json.asc", StringComparison.OrdinalIgnoreCase) + || relativePath.EndsWith(".json.sha256", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + documentUri = new Uri(advisoryRoot, relativePath); + return true; + } + private static Uri? ParseDocumentLink(XElement entry, XNamespace ns) { var linkElements = entry.Elements(ns + "link"); diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptions.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptions.cs index ab0eedeac..2610a3ac4 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptions.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptions.cs @@ -7,18 +7,42 @@ namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration; public sealed class UbuntuConnectorOptions { + public static readonly Uri DefaultIndexUri = new("https://ubuntu.com/security/notices.json"); + public static readonly Uri DefaultNoticeDetailBaseUri = new("https://ubuntu.com/security/notices/"); + public const int MaxPageSize = 20; + public const string HttpClientName = "excititor.connector.ubuntu.catalog"; /// - /// Root index that lists Ubuntu CSAF channels. + /// Root index that lists Ubuntu security notices. /// - public Uri IndexUri { get; set; } = new("https://ubuntu.com/security/csaf/index.json"); + public Uri IndexUri { get; set; } = DefaultIndexUri; /// - /// Channels to include (e.g. stable, esm, lts). + /// Base URI used to resolve individual Ubuntu notice JSON documents. + /// + public Uri NoticeDetailBaseUri { get; set; } = DefaultNoticeDetailBaseUri; + + /// + /// Channels to include when a CSAF channel catalog is explicitly provided. /// public IList Channels { get; } = new List { "stable" }; + /// + /// Maximum number of live notices to evaluate during one fetch pass. + /// + public int MaxNoticesPerFetch { get; set; } = 60; + + /// + /// Number of notices requested per page from the live notices index. + /// + public int IndexPageSize { get; set; } = MaxPageSize; + + /// + /// Overlap window applied to the resume cursor when using the live notices feed. + /// + public TimeSpan ResumeOverlap { get; set; } = TimeSpan.FromDays(3); + /// /// Duration to cache discovery metadata. /// @@ -37,6 +61,11 @@ public sealed class UbuntuConnectorOptions /// public bool PersistOfflineSnapshot { get; set; } = true; + /// + /// When true, the loader infers channel catalog URLs from when the root index is unavailable. + /// + public bool AllowBuiltInSnapshotFallback { get; set; } = true; + /// /// Weight applied to Ubuntu-sourced statements during trust evaluation. /// @@ -74,6 +103,16 @@ public sealed class UbuntuConnectorOptions throw new InvalidOperationException("IndexUri must use HTTP or HTTPS."); } + if (NoticeDetailBaseUri is null || !NoticeDetailBaseUri.IsAbsoluteUri) + { + throw new InvalidOperationException("NoticeDetailBaseUri must be an absolute URI."); + } + + if (NoticeDetailBaseUri.Scheme is not ("http" or "https")) + { + throw new InvalidOperationException("NoticeDetailBaseUri must use HTTP or HTTPS."); + } + if (Channels.Count == 0) { throw new InvalidOperationException("At least one channel must be specified."); @@ -97,6 +136,21 @@ public sealed class UbuntuConnectorOptions throw new InvalidOperationException("MetadataCacheDuration must be positive."); } + if (MaxNoticesPerFetch <= 0 || MaxNoticesPerFetch > 200) + { + throw new InvalidOperationException("MaxNoticesPerFetch must be between 1 and 200."); + } + + if (IndexPageSize <= 0 || IndexPageSize > MaxPageSize) + { + throw new InvalidOperationException($"IndexPageSize must be between 1 and {MaxPageSize}."); + } + + if (ResumeOverlap < TimeSpan.Zero || ResumeOverlap > TimeSpan.FromDays(14)) + { + throw new InvalidOperationException("ResumeOverlap must be between 0 and 14 days."); + } + if (PreferOfflineSnapshot && string.IsNullOrWhiteSpace(OfflineSnapshotPath)) { throw new InvalidOperationException("OfflineSnapshotPath must be provided when PreferOfflineSnapshot is enabled."); diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Ubuntu.CSAF/Internal/UbuntuNoticeModels.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Ubuntu.CSAF/Internal/UbuntuNoticeModels.cs new file mode 100644 index 000000000..733ca500c --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Ubuntu.CSAF/Internal/UbuntuNoticeModels.cs @@ -0,0 +1,473 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Text.Encodings.Web; +using System.Text.Json; + +namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Internal; + +internal static class UbuntuNoticeParser +{ + public static UbuntuNoticeIndexPage ParseIndex(ReadOnlySpan payload) + { + using var document = JsonDocument.Parse(payload.ToArray()); + var root = document.RootElement; + if (!root.TryGetProperty("notices", out var noticesElement) || noticesElement.ValueKind != JsonValueKind.Array) + { + return UbuntuNoticeIndexPage.Empty; + } + + var notices = new List(noticesElement.GetArrayLength()); + foreach (var noticeElement in noticesElement.EnumerateArray()) + { + var noticeId = GetString(noticeElement, "id"); + if (string.IsNullOrWhiteSpace(noticeId)) + { + continue; + } + + notices.Add(new UbuntuNoticeSummary( + noticeId.Trim(), + GetString(noticeElement, "title")?.Trim() ?? noticeId.Trim(), + ParseDate(noticeElement, "published") ?? DateTimeOffset.MinValue)); + } + + var offset = root.TryGetProperty("offset", out var offsetElement) && offsetElement.ValueKind == JsonValueKind.Number + ? offsetElement.GetInt32() + : 0; + + var limit = root.TryGetProperty("limit", out var limitElement) && limitElement.ValueKind == JsonValueKind.Number + ? limitElement.GetInt32() + : notices.Count; + + var totalResults = root.TryGetProperty("total_results", out var totalResultsElement) && totalResultsElement.ValueKind == JsonValueKind.Number + ? totalResultsElement.GetInt32() + : notices.Count; + + return new UbuntuNoticeIndexPage(offset, limit, totalResults, notices); + } + + public static UbuntuNoticeDocument ParseNotice(ReadOnlySpan payload) + { + using var document = JsonDocument.Parse(payload.ToArray()); + var root = document.RootElement; + + var noticeId = GetString(root, "id"); + if (string.IsNullOrWhiteSpace(noticeId)) + { + throw new InvalidOperationException("Ubuntu notice payload did not include an id."); + } + + var releaseVersions = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (root.TryGetProperty("releases", out var releasesElement) && releasesElement.ValueKind == JsonValueKind.Array) + { + foreach (var releaseElement in releasesElement.EnumerateArray()) + { + var codename = GetString(releaseElement, "codename"); + var version = GetString(releaseElement, "version"); + if (string.IsNullOrWhiteSpace(codename) || string.IsNullOrWhiteSpace(version)) + { + continue; + } + + releaseVersions[codename.Trim()] = version.Trim(); + } + } + + var packages = new List(); + if (root.TryGetProperty("release_packages", out var releasePackagesElement) && releasePackagesElement.ValueKind == JsonValueKind.Object) + { + foreach (var releaseProperty in releasePackagesElement.EnumerateObject()) + { + if (releaseProperty.Value.ValueKind != JsonValueKind.Array) + { + continue; + } + + foreach (var packageElement in releaseProperty.Value.EnumerateArray()) + { + var name = GetString(packageElement, "name"); + var version = GetString(packageElement, "version"); + if (string.IsNullOrWhiteSpace(name) || string.IsNullOrWhiteSpace(version)) + { + continue; + } + + releaseVersions.TryGetValue(releaseProperty.Name, out var releaseVersion); + packages.Add(new UbuntuReleasePackage( + releaseProperty.Name.Trim(), + releaseVersion, + name.Trim(), + version.Trim(), + GetString(packageElement, "pocket")?.Trim(), + packageElement.TryGetProperty("is_source", out var sourceElement) && sourceElement.ValueKind == JsonValueKind.True)); + } + } + } + + if (packages.Count == 0) + { + throw new InvalidOperationException($"Ubuntu notice {noticeId} did not include any release packages."); + } + + var cves = new HashSet(StringComparer.OrdinalIgnoreCase); + if (root.TryGetProperty("cves_ids", out var cvesIdsElement) && cvesIdsElement.ValueKind == JsonValueKind.Array) + { + foreach (var cveElement in cvesIdsElement.EnumerateArray()) + { + var cve = cveElement.GetString(); + if (!string.IsNullOrWhiteSpace(cve)) + { + cves.Add(cve.Trim()); + } + } + } + + if (root.TryGetProperty("cves", out var cvesElement) && cvesElement.ValueKind == JsonValueKind.Array) + { + foreach (var cveElement in cvesElement.EnumerateArray()) + { + var cve = cveElement.ValueKind == JsonValueKind.Object + ? GetString(cveElement, "id") + : cveElement.GetString(); + + if (!string.IsNullOrWhiteSpace(cve)) + { + cves.Add(cve.Trim()); + } + } + } + + var references = Array.Empty(); + if (root.TryGetProperty("references", out var referencesElement) && referencesElement.ValueKind == JsonValueKind.Array) + { + references = referencesElement.EnumerateArray() + .Select(TryParseReference) + .Where(static reference => reference is not null) + .Select(static reference => reference!) + .Where(static reference => !string.IsNullOrWhiteSpace(reference.Url)) + .DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) + .OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + var relatedNotices = Array.Empty(); + if (root.TryGetProperty("related_notices", out var relatedElement) && relatedElement.ValueKind == JsonValueKind.Array) + { + relatedNotices = relatedElement.EnumerateArray() + .Select(static item => item.GetString()) + .Where(static item => !string.IsNullOrWhiteSpace(item)) + .Select(static item => item!.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static item => item, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + return new UbuntuNoticeDocument( + noticeId.Trim(), + GetString(root, "title")?.Trim() ?? noticeId.Trim(), + GetString(root, "summary")?.Trim(), + GetString(root, "description")?.Trim(), + GetString(root, "instructions")?.Trim(), + ParseDate(root, "published") ?? DateTimeOffset.MinValue, + cves.OrderBy(static cve => cve, StringComparer.OrdinalIgnoreCase).ToArray(), + packages + .OrderBy(static package => package.Release, StringComparer.OrdinalIgnoreCase) + .ThenBy(static package => package.Name, StringComparer.OrdinalIgnoreCase) + .ThenBy(static package => package.Version, StringComparer.OrdinalIgnoreCase) + .ToArray(), + references, + relatedNotices); + } + + private static string? GetString(JsonElement element, string propertyName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (!element.TryGetProperty(propertyName, out var property) || property.ValueKind != JsonValueKind.String) + { + return null; + } + + return property.GetString(); + } + + private static UbuntuReference? TryParseReference(JsonElement referenceElement) + { + if (referenceElement.ValueKind == JsonValueKind.String) + { + var url = referenceElement.GetString(); + return string.IsNullOrWhiteSpace(url) + ? null + : new UbuntuReference(url.Trim(), null, null); + } + + if (referenceElement.ValueKind != JsonValueKind.Object) + { + return null; + } + + var normalizedUrl = GetString(referenceElement, "url")?.Trim() + ?? GetString(referenceElement, "href")?.Trim(); + if (string.IsNullOrWhiteSpace(normalizedUrl)) + { + return null; + } + + return new UbuntuReference( + normalizedUrl, + GetString(referenceElement, "summary")?.Trim() + ?? GetString(referenceElement, "description")?.Trim() + ?? GetString(referenceElement, "title")?.Trim(), + GetString(referenceElement, "category")?.Trim() + ?? GetString(referenceElement, "type")?.Trim()); + } + + private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) + { + var value = GetString(element, propertyName); + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed) + ? parsed.ToUniversalTime() + : (DateTimeOffset?)null; + } +} + +internal static class UbuntuNoticeCsafBuilder +{ + private const string DefaultNoticePagePrefix = "https://ubuntu.com/security/notices/"; + + public static byte[] Build(UbuntuNoticeDocument notice, Uri sourceUri) + { + var published = notice.Published == DateTimeOffset.MinValue + ? DateTimeOffset.UnixEpoch + : notice.Published; + + var detailText = !string.IsNullOrWhiteSpace(notice.Description) + ? notice.Description! + : notice.Summary ?? notice.Title; + + var buffer = new ArrayBufferWriter(); + using var writer = new Utf8JsonWriter(buffer, new JsonWriterOptions + { + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + Indented = false, + }); + + writer.WriteStartObject(); + + writer.WritePropertyName("document"); + writer.WriteStartObject(); + writer.WriteString("category", "csaf_security_advisory"); + writer.WriteString("csaf_version", "2.0"); + writer.WriteString("lang", "en"); + + writer.WritePropertyName("publisher"); + writer.WriteStartObject(); + writer.WriteString("name", "Canonical Ltd."); + writer.WriteString("category", "vendor"); + writer.WriteString("namespace", "https://ubuntu.com/security"); + writer.WriteEndObject(); + + writer.WritePropertyName("tracking"); + writer.WriteStartObject(); + writer.WriteString("id", notice.Id); + writer.WriteString("status", "final"); + writer.WriteString("version", "1"); + writer.WriteString("initial_release_date", published.ToString("O", CultureInfo.InvariantCulture)); + writer.WriteString("current_release_date", published.ToString("O", CultureInfo.InvariantCulture)); + writer.WriteEndObject(); + + writer.WriteString("title", notice.Title); + + writer.WritePropertyName("references"); + writer.WriteStartArray(); + + writer.WriteStartObject(); + writer.WriteString("category", "external"); + writer.WriteString("summary", "Ubuntu notice JSON"); + writer.WriteString("url", sourceUri.ToString()); + writer.WriteEndObject(); + + if (Uri.TryCreate($"{DefaultNoticePagePrefix}{notice.Id}", UriKind.Absolute, out var htmlUri)) + { + writer.WriteStartObject(); + writer.WriteString("summary", "Ubuntu notice page"); + writer.WriteString("url", htmlUri.ToString()); + writer.WriteEndObject(); + } + + foreach (var reference in notice.References) + { + writer.WriteStartObject(); + if (!string.IsNullOrWhiteSpace(reference.Category)) + { + writer.WriteString("category", reference.Category); + } + + if (!string.IsNullOrWhiteSpace(reference.Summary)) + { + writer.WriteString("summary", reference.Summary); + } + + writer.WriteString("url", reference.Url); + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + writer.WriteEndObject(); + + writer.WritePropertyName("product_tree"); + writer.WriteStartObject(); + writer.WritePropertyName("full_product_names"); + writer.WriteStartArray(); + foreach (var package in notice.Packages) + { + writer.WriteStartObject(); + writer.WriteString("product_id", BuildProductId(package)); + writer.WriteString("name", package.Name); + writer.WritePropertyName("product_identification_helper"); + writer.WriteStartObject(); + if (!string.IsNullOrWhiteSpace(package.ReleaseVersion)) + { + writer.WriteString("cpe", $"cpe:/o:canonical:ubuntu_linux:{package.ReleaseVersion}"); + } + + writer.WriteString("purl", BuildPurl(package)); + writer.WriteEndObject(); + writer.WriteEndObject(); + } + writer.WriteEndArray(); + writer.WriteEndObject(); + + writer.WritePropertyName("vulnerabilities"); + writer.WriteStartArray(); + foreach (var cve in notice.Cves) + { + writer.WriteStartObject(); + writer.WriteString("cve", cve); + writer.WriteString("title", $"{notice.Title} ({cve})"); + + writer.WritePropertyName("product_status"); + writer.WriteStartObject(); + writer.WritePropertyName("fixed"); + writer.WriteStartArray(); + foreach (var package in notice.Packages) + { + writer.WriteStringValue(BuildProductId(package)); + } + writer.WriteEndArray(); + writer.WriteEndObject(); + + writer.WritePropertyName("notes"); + writer.WriteStartArray(); + writer.WriteStartObject(); + writer.WriteString("category", "description"); + writer.WriteString("text", detailText); + writer.WriteEndObject(); + + if (!string.IsNullOrWhiteSpace(notice.Instructions)) + { + writer.WriteStartObject(); + writer.WriteString("category", "details"); + writer.WriteString("text", notice.Instructions); + writer.WriteEndObject(); + } + writer.WriteEndArray(); + + if (notice.RelatedNotices.Count > 0) + { + writer.WritePropertyName("references"); + writer.WriteStartArray(); + foreach (var relatedNotice in notice.RelatedNotices) + { + writer.WriteStartObject(); + writer.WriteString("summary", $"Related notice {relatedNotice}"); + writer.WriteString("url", $"{DefaultNoticePagePrefix}{relatedNotice}"); + writer.WriteEndObject(); + } + writer.WriteEndArray(); + } + + writer.WriteEndObject(); + } + writer.WriteEndArray(); + + writer.WriteEndObject(); + writer.Flush(); + return buffer.WrittenSpan.ToArray(); + } + + private static string BuildProductId(UbuntuReleasePackage package) + => $"ubuntu:{Sanitize(package.Release)}:{Sanitize(package.Name)}:{Sanitize(package.Version)}{(package.IsSource ? ":source" : string.Empty)}"; + + private static string BuildPurl(UbuntuReleasePackage package) + { + var qualifiers = new List(2) + { + $"distro={Uri.EscapeDataString(package.Release)}" + }; + + if (package.IsSource) + { + qualifiers.Add("source=true"); + } + + if (!string.IsNullOrWhiteSpace(package.Pocket)) + { + qualifiers.Add($"channel={Uri.EscapeDataString(package.Pocket!)}"); + } + + return $"pkg:deb/ubuntu/{package.Name}@{Uri.EscapeDataString(package.Version)}?{string.Join("&", qualifiers)}"; + } + + private static string Sanitize(string value) + { + var builder = new StringBuilder(value.Length); + foreach (var character in value) + { + builder.Append(char.IsLetterOrDigit(character) || character is '-' or '_' or '.' ? character : '_'); + } + + return builder.ToString(); + } +} + +internal sealed record UbuntuNoticeIndexPage(int Offset, int Limit, int TotalResults, IReadOnlyList Notices) +{ + public static UbuntuNoticeIndexPage Empty { get; } = new(0, 0, 0, Array.Empty()); +} + +internal sealed record UbuntuNoticeSummary(string Id, string Title, DateTimeOffset Published); + +internal sealed record UbuntuNoticeDocument( + string Id, + string Title, + string? Summary, + string? Description, + string? Instructions, + DateTimeOffset Published, + IReadOnlyList Cves, + IReadOnlyList Packages, + IReadOnlyList References, + IReadOnlyList RelatedNotices); + +internal sealed record UbuntuReleasePackage( + string Release, + string? ReleaseVersion, + string Name, + string Version, + string? Pocket, + bool IsSource); + +internal sealed record UbuntuReference(string Url, string? Summary, string? Category); diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Ubuntu.CSAF/Metadata/UbuntuCatalogLoader.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Ubuntu.CSAF/Metadata/UbuntuCatalogLoader.cs index e1586ab31..18c57c04d 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Ubuntu.CSAF/Metadata/UbuntuCatalogLoader.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Ubuntu.CSAF/Metadata/UbuntuCatalogLoader.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration; +using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Internal; using System; using System.Collections.Generic; using System.Collections.Immutable; @@ -17,6 +18,15 @@ namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata; public sealed class UbuntuCatalogLoader { public const string CachePrefix = "StellaOps.Excititor.Connectors.Ubuntu.CSAF.Index"; + private const string LiveNoticesChannelName = "notices"; + private static readonly HashSet KnownBuiltInChannels = new(StringComparer.OrdinalIgnoreCase) + { + "stable", + "esm", + "esm-apps", + "esm-infra", + "lts", + }; private readonly IHttpClientFactory _httpClientFactory; private readonly IMemoryCache _memoryCache; @@ -63,10 +73,6 @@ public sealed class UbuntuCatalogLoader if (options.PreferOfflineSnapshot) { entry = LoadFromOffline(options); - if (entry is null) - { - throw new InvalidOperationException("PreferOfflineSnapshot is enabled but no offline Ubuntu snapshot was found."); - } } else { @@ -74,6 +80,14 @@ public sealed class UbuntuCatalogLoader ?? LoadFromOffline(options); } + if (entry is null && options.AllowBuiltInSnapshotFallback) + { + entry = CreateBuiltInFallbackEntry(options); + _logger.LogWarning( + "Falling back to built-in Ubuntu CSAF catalog inference for {IndexUri} because live discovery and offline snapshots were unavailable.", + options.IndexUri); + } + if (entry is null) { throw new InvalidOperationException("Unable to load Ubuntu CSAF index from network or offline snapshot."); @@ -94,6 +108,82 @@ public sealed class UbuntuCatalogLoader } } + private CacheEntry CreateBuiltInFallbackEntry(UbuntuConnectorOptions options) + { + var generatedAt = _timeProvider.GetUtcNow(); + var channels = IsLiveNoticeIndex(options.IndexUri) + ? ImmutableArray.Create(new UbuChannelCatalog(LiveNoticesChannelName, options.IndexUri, Sha256: null, LastUpdated: generatedAt)) + : BuildFallbackChannels(options); + if (channels.IsDefaultOrEmpty) + { + throw new InvalidOperationException("Unable to infer Ubuntu CSAF channel catalogs for the configured channel set."); + } + + var metadata = new UbuntuCatalogMetadata(generatedAt, channels); + return new CacheEntry(metadata, generatedAt, generatedAt, options.MetadataCacheDuration, true); + } + + private static ImmutableArray BuildFallbackChannels(UbuntuConnectorOptions options) + { + var root = GetCatalogRoot(options.IndexUri); + var builder = ImmutableArray.CreateBuilder(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var channel in options.Channels) + { + if (string.IsNullOrWhiteSpace(channel)) + { + continue; + } + + var trimmedChannel = channel.Trim(); + if (!ShouldInferChannel(options, trimmedChannel)) + { + continue; + } + + if (!seen.Add(trimmedChannel)) + { + continue; + } + + var catalogUri = new Uri(root, $"{trimmedChannel.Trim('/')}/catalog.json"); + builder.Add(new UbuChannelCatalog(trimmedChannel, catalogUri, Sha256: null, LastUpdated: null)); + } + + return builder.ToImmutable(); + } + + private static bool ShouldInferChannel(UbuntuConnectorOptions options, string channel) + => !Uri.Compare(options.IndexUri, UbuntuConnectorOptions.DefaultIndexUri, UriComponents.HttpRequestUrl, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase).Equals(0) + || KnownBuiltInChannels.Contains(channel); + + private static Uri GetCatalogRoot(Uri indexUri) + { + var builder = new UriBuilder(indexUri) + { + Query = string.Empty, + Fragment = string.Empty, + }; + + var path = builder.Path; + if (path.EndsWith("/index.json", StringComparison.OrdinalIgnoreCase)) + { + builder.Path = path[..^"index.json".Length]; + } + else if (!path.EndsWith("/", StringComparison.Ordinal)) + { + var lastSlash = path.LastIndexOf('/'); + builder.Path = lastSlash >= 0 ? path[..(lastSlash + 1)] : "/"; + } + + if (!builder.Path.EndsWith("/", StringComparison.Ordinal)) + { + builder.Path += "/"; + } + + return builder.Uri; + } + private async Task TryFetchFromNetworkAsync(UbuntuConnectorOptions options, CancellationToken cancellationToken) { try @@ -103,7 +193,7 @@ public sealed class UbuntuCatalogLoader response.EnsureSuccessStatusCode(); var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - var metadata = ParseMetadata(payload, options.Channels); + var metadata = ParseMetadata(payload, options); var now = _timeProvider.GetUtcNow(); var entry = new CacheEntry(metadata, now, now, options.MetadataCacheDuration, false); @@ -148,7 +238,7 @@ public sealed class UbuntuCatalogLoader } } - private UbuntuCatalogMetadata ParseMetadata(string payload, IList channels) + private UbuntuCatalogMetadata ParseMetadata(string payload, UbuntuConnectorOptions options) { if (string.IsNullOrWhiteSpace(payload)) { @@ -162,7 +252,23 @@ public sealed class UbuntuCatalogLoader ? generated : _timeProvider.GetUtcNow(); - var channelSet = new HashSet(channels, StringComparer.OrdinalIgnoreCase); + if (root.TryGetProperty("notices", out var noticesElement) && noticesElement.ValueKind == JsonValueKind.Array) + { + var page = UbuntuNoticeParser.ParseIndex(System.Text.Encoding.UTF8.GetBytes(payload)); + var lastUpdated = page.Notices.Count == 0 + ? generatedAt + : page.Notices + .Where(static notice => notice.Published != DateTimeOffset.MinValue) + .Select(static notice => (DateTimeOffset?)notice.Published) + .DefaultIfEmpty(generatedAt) + .Max() ?? generatedAt; + + return new UbuntuCatalogMetadata( + generatedAt, + ImmutableArray.Create(new UbuChannelCatalog(LiveNoticesChannelName, options.IndexUri, Sha256: null, LastUpdated: lastUpdated))); + } + + var channelSet = new HashSet(options.Channels, StringComparer.OrdinalIgnoreCase); if (!root.TryGetProperty("channels", out var channelsElement) || channelsElement.ValueKind is not JsonValueKind.Array) { @@ -207,6 +313,9 @@ public sealed class UbuntuCatalogLoader return new UbuntuCatalogMetadata(generatedAt, builder.ToImmutable()); } + private static bool IsLiveNoticeIndex(Uri indexUri) + => indexUri.AbsolutePath.EndsWith("/notices.json", StringComparison.OrdinalIgnoreCase); + private void PersistSnapshotIfNeeded(UbuntuConnectorOptions options, UbuntuCatalogMetadata metadata, DateTimeOffset fetchedAt) { if (!options.PersistOfflineSnapshot || string.IsNullOrWhiteSpace(options.OfflineSnapshotPath)) diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Ubuntu.CSAF/UbuntuCsafConnector.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Ubuntu.CSAF/UbuntuCsafConnector.cs index 30915f1a4..e86830bc5 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Ubuntu.CSAF/UbuntuCsafConnector.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Connectors.Ubuntu.CSAF/UbuntuCsafConnector.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging; using StellaOps.Excititor.Connectors.Abstractions; using StellaOps.Excititor.Connectors.Abstractions.Trust; using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration; +using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Internal; using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata; using StellaOps.Excititor.Core; using StellaOps.Excititor.Core.Storage; @@ -114,93 +115,105 @@ public sealed class UbuntuCsafConnector : VexConnectorBase } } - var since = context.Since ?? state?.LastUpdated ?? DateTimeOffset.MinValue; - var latestTimestamp = state?.LastUpdated ?? since; + var hasResumeState = knownTokens.Length > 0; + var persistedSince = hasResumeState ? state?.LastUpdated : null; + var since = context.Since ?? persistedSince ?? DateTimeOffset.MinValue; + var effectiveSince = IsLiveNoticeMode() + ? ApplyResumeOverlap(context.Since ?? persistedSince, _options.ResumeOverlap) ?? DateTimeOffset.MinValue + : since; + var latestTimestamp = persistedSince ?? since; var stateChanged = false; - foreach (var channel in _catalog.Metadata.Channels) + await foreach (var entry in EnumerateResourcesAsync(effectiveSince, cancellationToken).ConfigureAwait(false)) { - await foreach (var entry in EnumerateChannelResourcesAsync(channel, cancellationToken).ConfigureAwait(false)) + var entryTimestamp = entry.LastModified ?? _catalog.Metadata.GeneratedAt; + if (entryTimestamp <= effectiveSince) { - var entryTimestamp = entry.LastModified ?? channel.LastUpdated ?? _catalog.Metadata.GeneratedAt; - if (entryTimestamp <= since) - { - if (entryTimestamp > latestTimestamp) - { - latestTimestamp = entryTimestamp; - } - - continue; - } - - var expectedDigest = entry.Sha256 is null ? null : NormalizeDigest(entry.Sha256); - if (expectedDigest is not null && digestSet.Contains(expectedDigest)) - { - if (entryTimestamp > latestTimestamp) - { - latestTimestamp = entryTimestamp; - } - - continue; - } - - etagMap.TryGetValue(entry.DocumentUri.ToString(), out var knownEtag); - - var download = await DownloadDocumentAsync(entry, knownEtag, cancellationToken).ConfigureAwait(false); - if (download is null) - { - if (entryTimestamp > latestTimestamp) - { - latestTimestamp = entryTimestamp; - } - - continue; - } - - var document = download.Document; - if (!digestSet.Add(document.Digest)) - { - if (entryTimestamp > latestTimestamp) - { - latestTimestamp = entryTimestamp; - } - - continue; - } - - await context.RawSink.StoreAsync(document, cancellationToken).ConfigureAwait(false); - if (tokenSet.Add(document.Digest)) - { - tokenList.Add(document.Digest); - } - - if (!string.IsNullOrWhiteSpace(download.ETag)) - { - var etagValue = download.ETag!; - etagMap[entry.DocumentUri.ToString()] = etagValue; - var etagToken = CreateEtagToken(entry.DocumentUri, etagValue); - if (tokenSet.Add(etagToken)) - { - tokenList.Add(etagToken); - } - } - - stateChanged = true; if (entryTimestamp > latestTimestamp) { latestTimestamp = entryTimestamp; } - yield return document; + continue; } + + var expectedDigest = entry.Sha256 is null ? null : NormalizeDigest(entry.Sha256); + if (expectedDigest is not null && digestSet.Contains(expectedDigest)) + { + if (entryTimestamp > latestTimestamp) + { + latestTimestamp = entryTimestamp; + } + + continue; + } + + etagMap.TryGetValue(entry.DocumentUri.ToString(), out var knownEtag); + + var download = await DownloadDocumentAsync(entry, knownEtag, cancellationToken).ConfigureAwait(false); + if (download is null) + { + if (entryTimestamp > latestTimestamp) + { + latestTimestamp = entryTimestamp; + } + + continue; + } + + var document = download.Document; + if (!digestSet.Add(document.Digest)) + { + if (entryTimestamp > latestTimestamp) + { + latestTimestamp = entryTimestamp; + } + + continue; + } + + await context.RawSink.StoreAsync(document, cancellationToken).ConfigureAwait(false); + if (tokenSet.Add(document.Digest)) + { + tokenList.Add(document.Digest); + } + + if (!string.IsNullOrWhiteSpace(download.ETag)) + { + var etagValue = download.ETag!; + etagMap[entry.DocumentUri.ToString()] = etagValue; + var etagToken = CreateEtagToken(entry.DocumentUri, etagValue); + if (tokenSet.Add(etagToken)) + { + tokenList.Add(etagToken); + } + } + + stateChanged = true; + if (entryTimestamp > latestTimestamp) + { + latestTimestamp = entryTimestamp; + } + + yield return document; } - if (stateChanged || latestTimestamp > (state?.LastUpdated ?? DateTimeOffset.MinValue)) + if (stateChanged || latestTimestamp > (persistedSince ?? DateTimeOffset.MinValue)) { - var newState = new VexConnectorState( + var baseState = state ?? new VexConnectorState( Descriptor.Id, - latestTimestamp == DateTimeOffset.MinValue ? state?.LastUpdated : latestTimestamp, - tokenList.ToImmutableArray()); + null, + ImmutableArray.Empty, + ImmutableDictionary.Empty, + null, + 0, + null, + null); + var newState = baseState with + { + LastUpdated = latestTimestamp == DateTimeOffset.MinValue ? baseState.LastUpdated : latestTimestamp, + DocumentDigests = tokenList.ToImmutableArray(), + }; await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false); } @@ -211,6 +224,122 @@ public sealed class UbuntuCsafConnector : VexConnectorBase public UbuntuCatalogResult? GetCachedCatalog() => _catalog; + private async IAsyncEnumerable EnumerateResourcesAsync(DateTimeOffset since, [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (IsLiveNoticeMode()) + { + await foreach (var entry in EnumerateNoticeResourcesAsync(since, cancellationToken).ConfigureAwait(false)) + { + yield return entry; + } + + yield break; + } + + if (_catalog is null) + { + yield break; + } + + foreach (var channel in _catalog.Metadata.Channels) + { + await foreach (var entry in EnumerateChannelResourcesAsync(channel, cancellationToken).ConfigureAwait(false)) + { + yield return entry; + } + } + } + + private async IAsyncEnumerable EnumerateNoticeResourcesAsync(DateTimeOffset since, [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (_options is null) + { + yield break; + } + + var client = _httpClientFactory.CreateClient(UbuntuConnectorOptions.HttpClientName); + var entries = new List(_options.MaxNoticesPerFetch); + var seenNoticeIds = new HashSet(StringComparer.OrdinalIgnoreCase); + var pageSize = Math.Min(_options.IndexPageSize, UbuntuConnectorOptions.MaxPageSize); + + for (var offset = 0; entries.Count < _options.MaxNoticesPerFetch; offset += pageSize) + { + cancellationToken.ThrowIfCancellationRequested(); + + var indexPageUri = BuildIndexPageUri(_options.IndexUri, offset, pageSize); + try + { + using var response = await client.GetAsync(indexPageUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + var payload = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + var page = UbuntuNoticeParser.ParseIndex(payload); + if (page.Notices.Count == 0) + { + yield break; + } + + foreach (var notice in page.Notices) + { + if (!seenNoticeIds.Add(notice.Id)) + { + continue; + } + + if (notice.Published != DateTimeOffset.MinValue && notice.Published <= since) + { + continue; + } + + entries.Add(new UbuntuCatalogEntry( + Channel: "notices", + AdvisoryId: notice.Id, + DocumentUri: BuildNoticeDocumentUri(notice.Id), + Sha256: null, + ETag: null, + LastModified: notice.Published == DateTimeOffset.MinValue ? null : notice.Published, + Title: notice.Title, + Version: null)); + + if (entries.Count >= _options.MaxNoticesPerFetch) + { + break; + } + } + + var reachedEnd = page.Offset + page.Limit >= page.TotalResults || page.Notices.Count < pageSize; + if (reachedEnd) + { + break; + } + + if (page.Notices.All(static notice => notice.Published == DateTimeOffset.MinValue) || + page.Notices.All(notice => notice.Published != DateTimeOffset.MinValue && notice.Published <= since)) + { + break; + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LogConnectorEvent(LogLevel.Warning, "fetch.notices.failure", "Failed to enumerate Ubuntu notices feed page.", new Dictionary + { + ["uri"] = indexPageUri.ToString(), + ["offset"] = offset, + ["limit"] = pageSize, + }, ex); + yield break; + } + } + + foreach (var entry in entries + .OrderBy(static entry => entry.LastModified ?? DateTimeOffset.MinValue) + .ThenBy(static entry => entry.AdvisoryId, StringComparer.OrdinalIgnoreCase)) + { + cancellationToken.ThrowIfCancellationRequested(); + yield return entry; + } + } + private async IAsyncEnumerable EnumerateChannelResourcesAsync(UbuChannelCatalog channel, [EnumeratorCancellation] CancellationToken cancellationToken) { var client = _httpClientFactory.CreateClient(UbuntuConnectorOptions.HttpClientName); @@ -302,6 +431,11 @@ public sealed class UbuntuCsafConnector : VexConnectorBase private async Task DownloadDocumentAsync(UbuntuCatalogEntry entry, string? knownEtag, CancellationToken cancellationToken) { + if (string.Equals(entry.Channel, "notices", StringComparison.OrdinalIgnoreCase)) + { + return await DownloadNoticeDocumentAsync(entry, knownEtag, cancellationToken).ConfigureAwait(false); + } + var client = _httpClientFactory.CreateClient(UbuntuConnectorOptions.HttpClientName); using var request = new HttpRequestMessage(HttpMethod.Get, entry.DocumentUri); if (!string.IsNullOrWhiteSpace(knownEtag)) @@ -402,6 +536,78 @@ public sealed class UbuntuCsafConnector : VexConnectorBase } } + private async Task DownloadNoticeDocumentAsync(UbuntuCatalogEntry entry, string? knownEtag, CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(UbuntuConnectorOptions.HttpClientName); + using var request = new HttpRequestMessage(HttpMethod.Get, entry.DocumentUri); + if (!string.IsNullOrWhiteSpace(knownEtag)) + { + request.Headers.IfNoneMatch.TryParseAdd(EnsureQuoted(knownEtag)); + } + + HttpResponseMessage? response = null; + try + { + response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + if (response.StatusCode == HttpStatusCode.NotModified) + { + LogConnectorEvent(LogLevel.Debug, "fetch.notice.not_modified", "Ubuntu notice JSON not modified per ETag.", new Dictionary + { + ["uri"] = entry.DocumentUri.ToString(), + ["etag"] = knownEtag, + }); + return null; + } + + response.EnsureSuccessStatusCode(); + + var payload = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + var notice = UbuntuNoticeParser.ParseNotice(payload); + var csafPayload = UbuntuNoticeCsafBuilder.Build(notice, entry.DocumentUri); + + var etagHeader = response.Headers.ETag?.Tag; + var etagValue = !string.IsNullOrWhiteSpace(etagHeader) + ? Unquote(etagHeader!) + : null; + + var metadata = BuildMetadata(builder => + { + builder.Add("ubuntu.channel", entry.Channel); + builder.Add("ubuntu.uri", entry.DocumentUri.ToString()); + builder.Add("ubuntu.sourceUri", entry.DocumentUri.ToString()); + builder.Add("ubuntu.advisoryId", notice.Id); + builder.Add("ubuntu.title", notice.Title); + if (notice.Published != DateTimeOffset.MinValue) + { + builder.Add("ubuntu.lastModified", notice.Published.ToString("O", CultureInfo.InvariantCulture)); + } + + if (!string.IsNullOrWhiteSpace(etagValue)) + { + builder.Add("ubuntu.etag", etagValue!); + } + + AddProvenanceMetadata(builder); + }); + + var document = CreateRawDocument(VexDocumentFormat.Csaf, entry.DocumentUri, csafPayload, metadata); + return new DownloadResult(document, etagValue); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LogConnectorEvent(LogLevel.Warning, "fetch.notice.failure", "Failed to download or synthesize Ubuntu notice document.", new Dictionary + { + ["uri"] = entry.DocumentUri.ToString(), + ["advisoryId"] = entry.AdvisoryId, + }, ex); + return null; + } + finally + { + response?.Dispose(); + } + } + private VexProvider BuildProvider(UbuntuConnectorOptions options, UbuntuCatalogResult? catalog) { var baseUris = new List { options.IndexUri }; @@ -577,6 +783,67 @@ public sealed class UbuntuCsafConnector : VexConnectorBase private static string CreateEtagToken(Uri uri, string etag) => $"{EtagTokenPrefix}{uri}|{etag}"; + private bool IsLiveNoticeMode() + => _options is not null && IsLiveNoticeMode(_options.IndexUri); + + private static bool IsLiveNoticeMode(Uri indexUri) + => indexUri.AbsolutePath.EndsWith("/notices.json", StringComparison.OrdinalIgnoreCase); + + private static DateTimeOffset? ApplyResumeOverlap(DateTimeOffset? since, TimeSpan overlap) + { + if (since is null || overlap <= TimeSpan.Zero) + { + return since; + } + + var lowerBound = DateTimeOffset.MinValue.Add(overlap); + if (since.Value <= lowerBound) + { + return DateTimeOffset.MinValue; + } + + return since.Value - overlap; + } + + private Uri BuildNoticeDocumentUri(string noticeId) + { + if (_options is null) + { + throw new InvalidOperationException("Connector options must be available before building notice URIs."); + } + + var path = noticeId.EndsWith(".json", StringComparison.OrdinalIgnoreCase) + ? noticeId + : $"{noticeId}.json"; + + return new Uri(_options.NoticeDetailBaseUri, path); + } + + private static Uri BuildIndexPageUri(Uri indexUri, int offset, int limit) + { + var baseUri = indexUri.GetLeftPart(UriPartial.Path); + var parameters = new List(); + if (!string.IsNullOrWhiteSpace(indexUri.Query)) + { + foreach (var segment in indexUri.Query.TrimStart('?').Split('&', StringSplitOptions.RemoveEmptyEntries)) + { + var separator = segment.IndexOf('='); + var key = separator >= 0 ? segment[..separator] : segment; + if (key.Equals("offset", StringComparison.OrdinalIgnoreCase) || + key.Equals("limit", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + parameters.Add(segment); + } + } + + parameters.Add($"offset={offset.ToString(CultureInfo.InvariantCulture)}"); + parameters.Add($"limit={limit.ToString(CultureInfo.InvariantCulture)}"); + return new Uri($"{baseUri}?{string.Join("&", parameters)}", UriKind.Absolute); + } + private static bool TryParseEtagToken(string token, out string uri, out string etag) { uri = string.Empty; diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Core/Evidence/VexEvidenceServiceCollectionExtensions.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Core/Evidence/VexEvidenceServiceCollectionExtensions.cs index 875c48af0..d490c10a7 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Core/Evidence/VexEvidenceServiceCollectionExtensions.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Core/Evidence/VexEvidenceServiceCollectionExtensions.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using StellaOps.Attestation; +using System.Linq; namespace StellaOps.Excititor.Core.Evidence; @@ -21,13 +22,16 @@ public static class VexEvidenceServiceCollectionExtensions services.Configure(_ => { }); } - services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); - services.TryAddSingleton(); - services.TryAddSingleton(); + + if (services.Any(static descriptor => descriptor.ServiceType == typeof(IVexEvidenceLinkStore))) + { + services.TryAddSingleton(); + services.TryAddSingleton(); + } return services; } diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj b/src/Concelier/__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj index f9210df0f..3c166f109 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj @@ -12,11 +12,13 @@ + + @@ -26,5 +28,6 @@ + diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Core/TASKS.md b/src/Concelier/__Libraries/StellaOps.Excititor.Core/TASKS.md index 3b9d0e531..3a9a0983e 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Core/TASKS.md +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Core/TASKS.md @@ -15,3 +15,5 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | VEX-LINK-AUTOLINK-0001 | DONE | SPRINT_20260113_003_001 - Auto-linking pipeline. | | VEX-LINK-VALIDATION-0001 | DONE | SPRINT_20260113_003_001 - DSSE signature validation. | | VEX-LINK-DI-0001 | DONE | SPRINT_20260113_003_001 - DI registration and options. | +| REALPLAN-007-C | DONE | 2026-04-20: Replaced the verification cache stub with a real Valkey implementation, auto-wired cryptography for verification hosts, and removed the default in-memory/offline issuer-directory fallback from live verification registration. | +| REALPLAN-007-D | DONE | 2026-04-20: Removed the live evidence-linking in-memory registration path so runtime linking only activates when an explicitly registered real store is present. | diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Core/Verification/IVexSignatureVerifierV2.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Core/Verification/IVexSignatureVerifierV2.cs index 4773d17de..0e7d3364b 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Core/Verification/IVexSignatureVerifierV2.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Core/Verification/IVexSignatureVerifierV2.cs @@ -56,15 +56,13 @@ public interface IVexSignatureVerifierV2 public interface IVerificationCacheService { /// - /// Try to get a cached verification result. + /// Get a cached verification result. /// /// Cache key. - /// Cached result if found. /// Cancellation token. - /// True if cache hit. - Task TryGetAsync( + /// The cached result when found; otherwise . + Task GetAsync( string key, - out VexSignatureVerificationResult? result, CancellationToken ct = default); /// diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Core/Verification/ProductionVexSignatureVerifier.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Core/Verification/ProductionVexSignatureVerifier.cs index 590ce8cac..2ea317311 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Core/Verification/ProductionVexSignatureVerifier.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Core/Verification/ProductionVexSignatureVerifier.cs @@ -69,15 +69,19 @@ public sealed class ProductionVexSignatureVerifier : IVexSignatureVerifierV2 { // 1. Check cache var cacheKey = ComputeCacheKey(document.Digest, context.CryptoProfile); - if (_cache != null && await _cache.TryGetAsync(cacheKey, out var cached, ct)) + if (_cache != null) { - _logger.LogDebug( - "Cache hit for document {Digest} with profile {Profile}", - document.Digest, - context.CryptoProfile); + var cached = await _cache.GetAsync(cacheKey, ct).ConfigureAwait(false); + if (cached is not null) + { + _logger.LogDebug( + "Cache hit for document {Digest} with profile {Profile}", + document.Digest, + context.CryptoProfile); - VexVerificationMetrics.RecordCacheHit(); - return cached! with { VerifiedAt = DateTimeOffset.UtcNow }; + VexVerificationMetrics.RecordCacheHit(); + return cached with { VerifiedAt = DateTimeOffset.UtcNow }; + } } VexVerificationMetrics.RecordCacheMiss(); diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Core/Verification/VerificationCacheService.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Core/Verification/VerificationCacheService.cs index e76e58654..86a7f3a9d 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Core/Verification/VerificationCacheService.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Core/Verification/VerificationCacheService.cs @@ -1,12 +1,13 @@ // InMemoryVerificationCacheService - In-memory cache for verification results // Part of SPRINT_1227_0004_0001: Activate VEX Signature Verification Pipeline - using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using StackExchange.Redis; using System; using System.Collections.Concurrent; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -35,19 +36,16 @@ public sealed class InMemoryVerificationCacheService : IVerificationCacheService } /// - public Task TryGetAsync( + public Task GetAsync( string key, - out VexSignatureVerificationResult? result, CancellationToken ct = default) { if (_cache.TryGetValue(key, out var cached) && cached is VexSignatureVerificationResult cachedResult) { - result = cachedResult; - return Task.FromResult(true); + return Task.FromResult(cachedResult); } - result = null; - return Task.FromResult(false); + return Task.FromResult(null); } /// @@ -112,65 +110,238 @@ public sealed class InMemoryVerificationCacheService : IVerificationCacheService } /// -/// Stub implementation for Valkey-backed verification cache. -/// Requires StackExchange.Redis or similar Valkey client. +/// Valkey-backed verification cache. /// -public sealed class ValkeyVerificationCacheService : IVerificationCacheService +public sealed class ValkeyVerificationCacheService : IVerificationCacheService, IAsyncDisposable { + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + + private readonly string _connectionString; private readonly ILogger _logger; + private readonly int? _database; private readonly string _keyPrefix; + private readonly SemaphoreSlim _connectionLock = new(1, 1); + private readonly Func> _connectionFactory; + + private IConnectionMultiplexer? _connection; + private bool _disposed; public ValkeyVerificationCacheService( + string connectionString, ILogger logger, - string keyPrefix = "trust-verdict:") + int? database = null, + string keyPrefix = "excititor:vex-verification:", + Func>? connectionFactory = null) { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionString); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _connectionString = connectionString; + _database = database; _keyPrefix = keyPrefix; + _connectionFactory = connectionFactory ?? DefaultConnectionFactory; } /// - public Task TryGetAsync( + public async Task GetAsync( string key, - out VexSignatureVerificationResult? result, CancellationToken ct = default) { - // TODO: Implement Valkey/Redis lookup - // var db = _valkey.GetDatabase(); - // var value = await db.StringGetAsync($"{_keyPrefix}{key}"); - // if (value.IsNullOrEmpty) { result = null; return false; } - // result = JsonSerializer.Deserialize(value!); - // return true; + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ct.ThrowIfCancellationRequested(); - _logger.LogDebug("Valkey cache not implemented - cache miss for {Key}", key); - result = null; - return Task.FromResult(false); + try + { + var db = await GetDatabaseAsync(ct).ConfigureAwait(false); + var payload = await db.StringGetAsync(GetEntryKey(key)).ConfigureAwait(false); + + if (payload.IsNullOrEmpty) + { + return null; + } + + return JsonSerializer.Deserialize(payload.ToString(), JsonOptions); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to read verification cache entry {Key} from Valkey", key); + return null; + } } /// - public Task SetAsync( + public async Task SetAsync( string key, VexSignatureVerificationResult result, TimeSpan ttl, CancellationToken ct = default) { - // TODO: Implement Valkey/Redis storage - // var db = _valkey.GetDatabase(); - // var value = JsonSerializer.Serialize(result); - // await db.StringSetAsync($"{_keyPrefix}{key}", value, ttl); + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentNullException.ThrowIfNull(result); + ct.ThrowIfCancellationRequested(); - _logger.LogDebug("Valkey cache not implemented - skipping cache for {Key}", key); - return Task.CompletedTask; + try + { + var db = await GetDatabaseAsync(ct).ConfigureAwait(false); + var entryKey = GetEntryKey(key); + var payload = JsonSerializer.Serialize(result, JsonOptions); + + await db.StringSetAsync(entryKey, payload, ttl).ConfigureAwait(false); + + if (!string.IsNullOrWhiteSpace(result.IssuerId)) + { + var issuerIndexKey = GetIssuerIndexKey(result.IssuerId); + var indexTtl = ttl > TimeSpan.FromDays(1) ? ttl : TimeSpan.FromDays(1); + + await db.SetAddAsync(issuerIndexKey, entryKey).ConfigureAwait(false); + await db.KeyExpireAsync(issuerIndexKey, indexTtl).ConfigureAwait(false); + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to write verification cache entry {Key} to Valkey", key); + } } /// - public Task InvalidateByIssuerAsync( + public async Task InvalidateByIssuerAsync( string issuerId, CancellationToken ct = default) { - // TODO: Implement bulk invalidation - // This would use Redis SCAN to find matching keys or maintain a secondary index + ArgumentException.ThrowIfNullOrWhiteSpace(issuerId); + ct.ThrowIfCancellationRequested(); - _logger.LogDebug("Valkey cache invalidation not implemented for issuer {IssuerId}", issuerId); - return Task.CompletedTask; + try + { + var db = await GetDatabaseAsync(ct).ConfigureAwait(false); + var issuerIndexKey = GetIssuerIndexKey(issuerId); + var members = await db.SetMembersAsync(issuerIndexKey).ConfigureAwait(false); + + foreach (var member in members) + { + if (member.IsNullOrEmpty) + { + continue; + } + + await db.KeyDeleteAsync(member.ToString()).ConfigureAwait(false); + } + + await db.KeyDeleteAsync(issuerIndexKey).ConfigureAwait(false); + + _logger.LogInformation( + "Invalidated {Count} verification cache entries for issuer {IssuerId}", + members.Length, + issuerId); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to invalidate verification cache for issuer {IssuerId}", issuerId); + } } + + public async ValueTask DisposeAsync() + { + if (_disposed) + { + return; + } + + _disposed = true; + + if (_connection is not null) + { + await _connection.CloseAsync().ConfigureAwait(false); + _connection.Dispose(); + } + + _connectionLock.Dispose(); + } + + private async Task GetDatabaseAsync(CancellationToken cancellationToken) + { + ObjectDisposedException.ThrowIf(_disposed, this); + + if (_connection is not null && _connection.IsConnected) + { + return _connection.GetDatabase(_database ?? -1); + } + + await _connectionLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_connection is null || !_connection.IsConnected) + { + if (_connection is not null) + { + await _connection.CloseAsync().ConfigureAwait(false); + _connection.Dispose(); + } + + var options = ConfigurationOptions.Parse(_connectionString); + options.AbortOnConnectFail = false; + options.ClientName ??= "stellaops-excititor-verification-cache"; + if (_database.HasValue) + { + options.DefaultDatabase = _database.Value; + } + + _connection = await WaitWithCancellationAsync( + _connectionFactory(options), + cancellationToken).ConfigureAwait(false); + } + } + finally + { + _connectionLock.Release(); + } + + return _connection.GetDatabase(_database ?? -1); + } + + private string GetEntryKey(string key) => $"{_keyPrefix}{key}"; + + private string GetIssuerIndexKey(string issuerId) => $"{_keyPrefix}issuer:{issuerId}"; + + private static async Task WaitWithCancellationAsync( + Task task, + CancellationToken cancellationToken) + { + if (task.IsCompleted || !cancellationToken.CanBeCanceled) + { + return await task.ConfigureAwait(false); + } + + if (cancellationToken.IsCancellationRequested) + { + throw new OperationCanceledException(cancellationToken); + } + + var cancellationSignal = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var registration = cancellationToken.Register( + static state => ((TaskCompletionSource)state!).TrySetResult(true), + cancellationSignal); + + var completed = await Task.WhenAny(task, cancellationSignal.Task).ConfigureAwait(false); + if (completed == cancellationSignal.Task) + { + throw new OperationCanceledException(cancellationToken); + } + + return await task.ConfigureAwait(false); + } + + private static async Task DefaultConnectionFactory(ConfigurationOptions options) + => await ConnectionMultiplexer.ConnectAsync(options).ConfigureAwait(false); } diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Core/Verification/VexSignatureVerifierOptions.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Core/Verification/VexSignatureVerifierOptions.cs index 3b2ae193b..ec3d9a4ba 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Core/Verification/VexSignatureVerifierOptions.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Core/Verification/VexSignatureVerifierOptions.cs @@ -71,6 +71,12 @@ public sealed class VexSignatureVerifierOptions /// public IssuerDirectoryClientOptions IssuerDirectory { get; set; } = new(); + /// + /// Optional Valkey-backed verification cache configuration. + /// When omitted, verification executes without a cache. + /// + public ValkeyVerificationCacheOptions Valkey { get; set; } = new(); + /// /// Trust anchor configuration. /// @@ -105,10 +111,33 @@ public sealed class IssuerDirectoryClientOptions /// /// Whether to use offline mode. + /// Live webservice/runtime wiring does not seed an in-memory issuer directory; + /// configure a real IssuerDirectory URL instead when verification is enabled. /// public bool OfflineMode { get; set; } = false; } +/// +/// Optional Valkey-backed verification cache configuration. +/// +public sealed class ValkeyVerificationCacheOptions +{ + /// + /// Valkey/Redis connection string. Leave empty to disable caching. + /// + public string? ConnectionString { get; set; } + + /// + /// Optional database override. When omitted, the connection-string default is used. + /// + public int? Database { get; set; } + + /// + /// Cache key prefix. + /// + public string KeyPrefix { get; set; } = "excititor:vex-verification:"; +} + /// /// Trust anchor configuration for keyless verification. /// diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Core/Verification/VexVerificationServiceCollectionExtensions.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Core/Verification/VexVerificationServiceCollectionExtensions.cs index 8f16fe7fd..b43238c58 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Core/Verification/VexVerificationServiceCollectionExtensions.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Core/Verification/VexVerificationServiceCollectionExtensions.cs @@ -1,13 +1,15 @@ // VexVerificationServiceCollectionExtensions - DI Registration for Verification Services // Part of SPRINT_1227_0004_0001: Activate VEX Signature Verification Pipeline - using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using StellaOps.Cryptography; +using StellaOps.Cryptography.DependencyInjection; using System; +using System.Linq; namespace StellaOps.Excititor.Core.Verification; @@ -33,25 +35,51 @@ public static class VexVerificationServiceCollectionExtensions services.Configure( configuration.GetSection(VexSignatureVerifierOptions.SectionName)); + if (!services.Any(descriptor => descriptor.ServiceType == typeof(ICryptoProviderRegistry))) + { + services.AddStellaOpsCrypto(); + } + // Register crypto profile selector services.TryAddSingleton(); - // Register cache service (in-memory by default) - services.TryAddSingleton(); + var valkeyConnectionString = configuration[$"{VexSignatureVerifierOptions.SectionName}:Valkey:ConnectionString"]; + if (!string.IsNullOrWhiteSpace(valkeyConnectionString)) + { + services.TryAddSingleton(sp => + { + var options = sp.GetRequiredService>().Value; + var logger = sp.GetRequiredService>(); - // Register IssuerDirectory client based on configuration + return new ValkeyVerificationCacheService( + options.Valkey.ConnectionString!, + logger, + options.Valkey.Database, + options.Valkey.KeyPrefix); + }); + } + + services.AddHttpClient("IssuerDirectory"); + + // Register IssuerDirectory client based on configuration. + // Live runtime no longer seeds a process-local in-memory issuer directory. services.TryAddSingleton(sp => { var options = sp.GetRequiredService>().Value; - var logger = sp.GetRequiredService>(); - if (options.IssuerDirectory.OfflineMode || string.IsNullOrEmpty(options.IssuerDirectory.ServiceUrl)) + if (!options.Enabled) { - // Use in-memory client for development/offline - return new InMemoryIssuerDirectoryClient(logger); + throw new InvalidOperationException( + "IIssuerDirectoryClient is only available when VEX signature verification is enabled."); + } + + if (options.IssuerDirectory.OfflineMode || string.IsNullOrWhiteSpace(options.IssuerDirectory.ServiceUrl)) + { + throw new InvalidOperationException( + "Enabled VEX signature verification requires VexSignatureVerification:IssuerDirectory:ServiceUrl. " + + "The live runtime no longer uses the in-memory/offline issuer-directory fallback."); } - // Use HTTP client for production var httpLogger = sp.GetRequiredService>(); var httpClientFactory = sp.GetRequiredService(); var httpClient = httpClientFactory.CreateClient("IssuerDirectory"); @@ -59,9 +87,17 @@ public static class VexVerificationServiceCollectionExtensions return new HttpIssuerDirectoryClient(httpClient, options.IssuerDirectory, httpLogger); }); - // Register verifier based on feature flag - // The actual registration happens at runtime to support feature flag - services.TryAddSingleton(); + services.TryAddSingleton(sp => + { + var cache = sp.GetService(); + + return new ProductionVexSignatureVerifier( + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService>(), + sp.GetRequiredService>(), + cache); + }); services.TryAddSingleton(sp => { @@ -93,12 +129,18 @@ public static class VexVerificationServiceCollectionExtensions // Base registration services.AddVexSignatureVerification(configuration); - // Replace in-memory cache with Valkey + // Replace any implicit cache wiring with an explicit Valkey-backed cache. services.RemoveAll(); services.AddSingleton(sp => { var logger = sp.GetRequiredService>(); - return new ValkeyVerificationCacheService(logger); + var options = sp.GetRequiredService>().Value; + + return new ValkeyVerificationCacheService( + valkeyConnectionString, + logger, + options.Valkey.Database, + options.Valkey.KeyPrefix); }); return services; diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Core/VexNormalizationServiceCollectionExtensions.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Core/VexNormalizationServiceCollectionExtensions.cs new file mode 100644 index 000000000..d308a3265 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Core/VexNormalizationServiceCollectionExtensions.cs @@ -0,0 +1,50 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using StellaOps.Excititor.Core.Storage; +using System.Collections.Immutable; + +namespace StellaOps.Excititor.Core; + +public static class VexNormalizationServiceCollectionExtensions +{ + public static IServiceCollection AddVexNormalization(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + + services.TryAddSingleton(static provider => + new VexNormalizerRegistry(provider.GetServices().ToImmutableArray())); + services.TryAddSingleton(); + + return services; + } +} + +public sealed class DefaultVexNormalizerRouter : IVexNormalizerRouter +{ + private readonly VexNormalizerRegistry _registry; + private readonly IVexProviderStore _providerStore; + + public DefaultVexNormalizerRouter( + VexNormalizerRegistry registry, + IVexProviderStore providerStore) + { + _registry = registry ?? throw new ArgumentNullException(nameof(registry)); + _providerStore = providerStore ?? throw new ArgumentNullException(nameof(providerStore)); + } + + public async ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + + var normalizer = _registry.Resolve(document); + if (normalizer is null) + { + throw new InvalidOperationException($"No IVexNormalizer implementation is registered for format '{document.Format}'."); + } + + var provider = await _providerStore.FindAsync(document.ProviderId, cancellationToken).ConfigureAwait(false) + ?? new VexProvider(document.ProviderId, document.ProviderId, VexProviderKind.Vendor); + + return await normalizer.NormalizeAsync(document, provider, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/AttestationRowEntityType.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/AttestationRowEntityType.cs new file mode 100644 index 000000000..153a5db35 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/AttestationRowEntityType.cs @@ -0,0 +1,151 @@ +// +using System; +using System.Reflection; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using StellaOps.Excititor.Persistence.EfCore.Models; + +#pragma warning disable 219, 612, 618 +#nullable disable + +namespace StellaOps.Excititor.Persistence.EfCore.CompiledModels +{ + [EntityFrameworkInternal] + public partial class AttestationRowEntityType + { + public static RuntimeEntityType Create(RuntimeModel model, RuntimeEntityType baseEntityType = null) + { + var runtimeEntityType = model.AddEntityType( + "StellaOps.Excititor.Persistence.EfCore.Models.AttestationRow", + typeof(AttestationRow), + baseEntityType, + propertyCount: 10, + unnamedIndexCount: 3, + keyCount: 1); + + var tenant = runtimeEntityType.AddProperty( + "Tenant", + typeof(string), + propertyInfo: typeof(AttestationRow).GetProperty("Tenant", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(AttestationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + afterSaveBehavior: PropertySaveBehavior.Throw); + tenant.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + tenant.AddAnnotation("Relational:ColumnName", "tenant"); + + var attestationId = runtimeEntityType.AddProperty( + "AttestationId", + typeof(string), + propertyInfo: typeof(AttestationRow).GetProperty("AttestationId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(AttestationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + afterSaveBehavior: PropertySaveBehavior.Throw); + attestationId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + attestationId.AddAnnotation("Relational:ColumnName", "attestation_id"); + + var attestedAt = runtimeEntityType.AddProperty( + "AttestedAt", + typeof(DateTime), + propertyInfo: typeof(AttestationRow).GetProperty("AttestedAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(AttestationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + attestedAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + attestedAt.AddAnnotation("Relational:ColumnName", "attested_at"); + + var createdAt = runtimeEntityType.AddProperty( + "CreatedAt", + typeof(DateTime), + propertyInfo: typeof(AttestationRow).GetProperty("CreatedAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(AttestationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + createdAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + createdAt.AddAnnotation("Relational:ColumnName", "created_at"); + createdAt.AddAnnotation("Relational:DefaultValueSql", "now()"); + + var dsseEnvelopeHash = runtimeEntityType.AddProperty( + "DsseEnvelopeHash", + typeof(string), + propertyInfo: typeof(AttestationRow).GetProperty("DsseEnvelopeHash", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(AttestationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + dsseEnvelopeHash.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + dsseEnvelopeHash.AddAnnotation("Relational:ColumnName", "dsse_envelope_hash"); + + var dsseEnvelopeJson = runtimeEntityType.AddProperty( + "DsseEnvelopeJson", + typeof(string), + propertyInfo: typeof(AttestationRow).GetProperty("DsseEnvelopeJson", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(AttestationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + dsseEnvelopeJson.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + dsseEnvelopeJson.AddAnnotation("Relational:ColumnName", "dsse_envelope_json"); + + var itemCount = runtimeEntityType.AddProperty( + "ItemCount", + typeof(int), + propertyInfo: typeof(AttestationRow).GetProperty("ItemCount", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(AttestationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: 0); + itemCount.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + itemCount.AddAnnotation("Relational:ColumnName", "item_count"); + + var manifestId = runtimeEntityType.AddProperty( + "ManifestId", + typeof(string), + propertyInfo: typeof(AttestationRow).GetProperty("ManifestId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(AttestationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + manifestId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + manifestId.AddAnnotation("Relational:ColumnName", "manifest_id"); + + var merkleRoot = runtimeEntityType.AddProperty( + "MerkleRoot", + typeof(string), + propertyInfo: typeof(AttestationRow).GetProperty("MerkleRoot", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(AttestationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + merkleRoot.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + merkleRoot.AddAnnotation("Relational:ColumnName", "merkle_root"); + + var metadata = runtimeEntityType.AddProperty( + "Metadata", + typeof(string), + propertyInfo: typeof(AttestationRow).GetProperty("Metadata", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(AttestationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd); + metadata.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + metadata.AddAnnotation("Relational:ColumnName", "metadata"); + metadata.AddAnnotation("Relational:ColumnType", "jsonb"); + metadata.AddAnnotation("Relational:DefaultValueSql", "'{}'::jsonb"); + + var key = runtimeEntityType.AddKey( + new[] { tenant, attestationId }); + runtimeEntityType.SetPrimaryKey(key); + key.AddAnnotation("Relational:Name", "attestations_pkey"); + + var index = runtimeEntityType.AddIndex( + new[] { tenant }); + index.AddAnnotation("Relational:Name", "idx_attestations_tenant"); + + var index0 = runtimeEntityType.AddIndex( + new[] { tenant, attestedAt }); + index0.AddAnnotation("Relational:Name", "idx_attestations_attested_at"); + + var index1 = runtimeEntityType.AddIndex( + new[] { tenant, manifestId }); + index1.AddAnnotation("Relational:Name", "idx_attestations_manifest_id"); + + return runtimeEntityType; + } + + public static void CreateAnnotations(RuntimeEntityType runtimeEntityType) + { + runtimeEntityType.AddAnnotation("Relational:FunctionName", null); + runtimeEntityType.AddAnnotation("Relational:Schema", "vex"); + runtimeEntityType.AddAnnotation("Relational:SqlQuery", null); + runtimeEntityType.AddAnnotation("Relational:TableName", "attestations"); + runtimeEntityType.AddAnnotation("Relational:ViewName", null); + runtimeEntityType.AddAnnotation("Relational:ViewSchema", null); + + Customize(runtimeEntityType); + } + + static partial void Customize(RuntimeEntityType runtimeEntityType); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/CalibrationAdjustmentEntityType.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/CalibrationAdjustmentEntityType.cs new file mode 100644 index 000000000..55049cb88 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/CalibrationAdjustmentEntityType.cs @@ -0,0 +1,179 @@ +// +using System; +using System.Reflection; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using StellaOps.Excititor.Persistence.EfCore.Models; + +#pragma warning disable 219, 612, 618 +#nullable disable + +namespace StellaOps.Excititor.Persistence.EfCore.CompiledModels +{ + [EntityFrameworkInternal] + public partial class CalibrationAdjustmentEntityType + { + public static RuntimeEntityType Create(RuntimeModel model, RuntimeEntityType baseEntityType = null) + { + var runtimeEntityType = model.AddEntityType( + "StellaOps.Excititor.Persistence.EfCore.Models.CalibrationAdjustment", + typeof(CalibrationAdjustment), + baseEntityType, + propertyCount: 14, + unnamedIndexCount: 1, + keyCount: 1); + + var id = runtimeEntityType.AddProperty( + "Id", + typeof(Guid), + propertyInfo: typeof(CalibrationAdjustment).GetProperty("Id", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationAdjustment).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + afterSaveBehavior: PropertySaveBehavior.Throw, + sentinel: new Guid("00000000-0000-0000-0000-000000000000")); + id.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + id.AddAnnotation("Relational:ColumnName", "id"); + id.AddAnnotation("Relational:DefaultValueSql", "gen_random_uuid()"); + + var accuracyAfter = runtimeEntityType.AddProperty( + "AccuracyAfter", + typeof(double), + propertyInfo: typeof(CalibrationAdjustment).GetProperty("AccuracyAfter", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationAdjustment).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: 0.0); + accuracyAfter.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + accuracyAfter.AddAnnotation("Relational:ColumnName", "accuracy_after"); + + var accuracyBefore = runtimeEntityType.AddProperty( + "AccuracyBefore", + typeof(double), + propertyInfo: typeof(CalibrationAdjustment).GetProperty("AccuracyBefore", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationAdjustment).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: 0.0); + accuracyBefore.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + accuracyBefore.AddAnnotation("Relational:ColumnName", "accuracy_before"); + + var delta = runtimeEntityType.AddProperty( + "Delta", + typeof(double), + propertyInfo: typeof(CalibrationAdjustment).GetProperty("Delta", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationAdjustment).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: 0.0); + delta.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + delta.AddAnnotation("Relational:ColumnName", "delta"); + + var manifestId = runtimeEntityType.AddProperty( + "ManifestId", + typeof(string), + propertyInfo: typeof(CalibrationAdjustment).GetProperty("ManifestId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationAdjustment).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + manifestId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + manifestId.AddAnnotation("Relational:ColumnName", "manifest_id"); + + var newCoverage = runtimeEntityType.AddProperty( + "NewCoverage", + typeof(double), + propertyInfo: typeof(CalibrationAdjustment).GetProperty("NewCoverage", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationAdjustment).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: 0.0); + newCoverage.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + newCoverage.AddAnnotation("Relational:ColumnName", "new_coverage"); + + var newProvenance = runtimeEntityType.AddProperty( + "NewProvenance", + typeof(double), + propertyInfo: typeof(CalibrationAdjustment).GetProperty("NewProvenance", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationAdjustment).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: 0.0); + newProvenance.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + newProvenance.AddAnnotation("Relational:ColumnName", "new_provenance"); + + var newReplayability = runtimeEntityType.AddProperty( + "NewReplayability", + typeof(double), + propertyInfo: typeof(CalibrationAdjustment).GetProperty("NewReplayability", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationAdjustment).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: 0.0); + newReplayability.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + newReplayability.AddAnnotation("Relational:ColumnName", "new_replayability"); + + var oldCoverage = runtimeEntityType.AddProperty( + "OldCoverage", + typeof(double), + propertyInfo: typeof(CalibrationAdjustment).GetProperty("OldCoverage", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationAdjustment).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: 0.0); + oldCoverage.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + oldCoverage.AddAnnotation("Relational:ColumnName", "old_coverage"); + + var oldProvenance = runtimeEntityType.AddProperty( + "OldProvenance", + typeof(double), + propertyInfo: typeof(CalibrationAdjustment).GetProperty("OldProvenance", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationAdjustment).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: 0.0); + oldProvenance.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + oldProvenance.AddAnnotation("Relational:ColumnName", "old_provenance"); + + var oldReplayability = runtimeEntityType.AddProperty( + "OldReplayability", + typeof(double), + propertyInfo: typeof(CalibrationAdjustment).GetProperty("OldReplayability", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationAdjustment).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: 0.0); + oldReplayability.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + oldReplayability.AddAnnotation("Relational:ColumnName", "old_replayability"); + + var reason = runtimeEntityType.AddProperty( + "Reason", + typeof(string), + propertyInfo: typeof(CalibrationAdjustment).GetProperty("Reason", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationAdjustment).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + reason.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + reason.AddAnnotation("Relational:ColumnName", "reason"); + + var sampleCount = runtimeEntityType.AddProperty( + "SampleCount", + typeof(int), + propertyInfo: typeof(CalibrationAdjustment).GetProperty("SampleCount", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationAdjustment).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: 0); + sampleCount.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + sampleCount.AddAnnotation("Relational:ColumnName", "sample_count"); + + var sourceId = runtimeEntityType.AddProperty( + "SourceId", + typeof(string), + propertyInfo: typeof(CalibrationAdjustment).GetProperty("SourceId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationAdjustment).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + sourceId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + sourceId.AddAnnotation("Relational:ColumnName", "source_id"); + + var key = runtimeEntityType.AddKey( + new[] { id }); + runtimeEntityType.SetPrimaryKey(key); + key.AddAnnotation("Relational:Name", "calibration_adjustments_pkey"); + + var index = runtimeEntityType.AddIndex( + new[] { manifestId }); + index.AddAnnotation("Relational:Name", "idx_calibration_adjustments_manifest"); + + return runtimeEntityType; + } + + public static void CreateAnnotations(RuntimeEntityType runtimeEntityType) + { + runtimeEntityType.AddAnnotation("Relational:FunctionName", null); + runtimeEntityType.AddAnnotation("Relational:Schema", "excititor"); + runtimeEntityType.AddAnnotation("Relational:SqlQuery", null); + runtimeEntityType.AddAnnotation("Relational:TableName", "calibration_adjustments"); + runtimeEntityType.AddAnnotation("Relational:ViewName", null); + runtimeEntityType.AddAnnotation("Relational:ViewSchema", null); + + Customize(runtimeEntityType); + } + + static partial void Customize(RuntimeEntityType runtimeEntityType); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/CalibrationManifestEntityType.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/CalibrationManifestEntityType.cs new file mode 100644 index 000000000..fe8f1a71c --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/CalibrationManifestEntityType.cs @@ -0,0 +1,160 @@ +// +using System; +using System.Reflection; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using StellaOps.Excititor.Persistence.EfCore.Models; + +#pragma warning disable 219, 612, 618 +#nullable disable + +namespace StellaOps.Excititor.Persistence.EfCore.CompiledModels +{ + [EntityFrameworkInternal] + public partial class CalibrationManifestEntityType + { + public static RuntimeEntityType Create(RuntimeModel model, RuntimeEntityType baseEntityType = null) + { + var runtimeEntityType = model.AddEntityType( + "StellaOps.Excititor.Persistence.EfCore.Models.CalibrationManifest", + typeof(CalibrationManifest), + baseEntityType, + propertyCount: 11, + unnamedIndexCount: 2, + keyCount: 1); + + var id = runtimeEntityType.AddProperty( + "Id", + typeof(Guid), + propertyInfo: typeof(CalibrationManifest).GetProperty("Id", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationManifest).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + afterSaveBehavior: PropertySaveBehavior.Throw, + sentinel: new Guid("00000000-0000-0000-0000-000000000000")); + id.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + id.AddAnnotation("Relational:ColumnName", "id"); + id.AddAnnotation("Relational:DefaultValueSql", "gen_random_uuid()"); + + var appliedAt = runtimeEntityType.AddProperty( + "AppliedAt", + typeof(DateTime?), + propertyInfo: typeof(CalibrationManifest).GetProperty("AppliedAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationManifest).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + appliedAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + appliedAt.AddAnnotation("Relational:ColumnName", "applied_at"); + + var createdAt = runtimeEntityType.AddProperty( + "CreatedAt", + typeof(DateTime), + propertyInfo: typeof(CalibrationManifest).GetProperty("CreatedAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationManifest).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + createdAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + createdAt.AddAnnotation("Relational:ColumnName", "created_at"); + createdAt.AddAnnotation("Relational:DefaultValueSql", "now()"); + + var epochEnd = runtimeEntityType.AddProperty( + "EpochEnd", + typeof(DateTime), + propertyInfo: typeof(CalibrationManifest).GetProperty("EpochEnd", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationManifest).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + epochEnd.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + epochEnd.AddAnnotation("Relational:ColumnName", "epoch_end"); + + var epochNumber = runtimeEntityType.AddProperty( + "EpochNumber", + typeof(int), + propertyInfo: typeof(CalibrationManifest).GetProperty("EpochNumber", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationManifest).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: 0); + epochNumber.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + epochNumber.AddAnnotation("Relational:ColumnName", "epoch_number"); + + var epochStart = runtimeEntityType.AddProperty( + "EpochStart", + typeof(DateTime), + propertyInfo: typeof(CalibrationManifest).GetProperty("EpochStart", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationManifest).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + epochStart.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + epochStart.AddAnnotation("Relational:ColumnName", "epoch_start"); + + var manifestDigest = runtimeEntityType.AddProperty( + "ManifestDigest", + typeof(string), + propertyInfo: typeof(CalibrationManifest).GetProperty("ManifestDigest", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationManifest).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + manifestDigest.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + manifestDigest.AddAnnotation("Relational:ColumnName", "manifest_digest"); + + var manifestId = runtimeEntityType.AddProperty( + "ManifestId", + typeof(string), + propertyInfo: typeof(CalibrationManifest).GetProperty("ManifestId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationManifest).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + manifestId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + manifestId.AddAnnotation("Relational:ColumnName", "manifest_id"); + + var metricsJson = runtimeEntityType.AddProperty( + "MetricsJson", + typeof(string), + propertyInfo: typeof(CalibrationManifest).GetProperty("MetricsJson", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationManifest).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + metricsJson.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + metricsJson.AddAnnotation("Relational:ColumnName", "metrics_json"); + metricsJson.AddAnnotation("Relational:ColumnType", "jsonb"); + + var signature = runtimeEntityType.AddProperty( + "Signature", + typeof(string), + propertyInfo: typeof(CalibrationManifest).GetProperty("Signature", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationManifest).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + signature.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + signature.AddAnnotation("Relational:ColumnName", "signature"); + + var tenant = runtimeEntityType.AddProperty( + "Tenant", + typeof(string), + propertyInfo: typeof(CalibrationManifest).GetProperty("Tenant", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CalibrationManifest).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + tenant.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + tenant.AddAnnotation("Relational:ColumnName", "tenant"); + + var key = runtimeEntityType.AddKey( + new[] { id }); + runtimeEntityType.SetPrimaryKey(key); + key.AddAnnotation("Relational:Name", "calibration_manifests_pkey"); + + var index = runtimeEntityType.AddIndex( + new[] { manifestId }, + unique: true); + index.AddAnnotation("Relational:Name", "calibration_manifests_manifest_id_key"); + + var index0 = runtimeEntityType.AddIndex( + new[] { tenant, epochNumber }, + unique: true); + index0.AddAnnotation("Relational:Name", "idx_calibration_tenant_epoch"); + + return runtimeEntityType; + } + + public static void CreateAnnotations(RuntimeEntityType runtimeEntityType) + { + runtimeEntityType.AddAnnotation("Relational:FunctionName", null); + runtimeEntityType.AddAnnotation("Relational:Schema", "excititor"); + runtimeEntityType.AddAnnotation("Relational:SqlQuery", null); + runtimeEntityType.AddAnnotation("Relational:TableName", "calibration_manifests"); + runtimeEntityType.AddAnnotation("Relational:ViewName", null); + runtimeEntityType.AddAnnotation("Relational:ViewSchema", null); + + Customize(runtimeEntityType); + } + + static partial void Customize(RuntimeEntityType runtimeEntityType); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/CheckpointMutationRowEntityType.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/CheckpointMutationRowEntityType.cs new file mode 100644 index 000000000..628ca36a9 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/CheckpointMutationRowEntityType.cs @@ -0,0 +1,198 @@ +// +using System; +using System.Reflection; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using StellaOps.Excititor.Persistence.EfCore.Models; + +#pragma warning disable 219, 612, 618 +#nullable disable + +namespace StellaOps.Excititor.Persistence.EfCore.CompiledModels +{ + [EntityFrameworkInternal] + public partial class CheckpointMutationRowEntityType + { + public static RuntimeEntityType Create(RuntimeModel model, RuntimeEntityType baseEntityType = null) + { + var runtimeEntityType = model.AddEntityType( + "StellaOps.Excititor.Persistence.EfCore.Models.CheckpointMutationRow", + typeof(CheckpointMutationRow), + baseEntityType, + propertyCount: 16, + unnamedIndexCount: 1, + keyCount: 1); + + var sequenceNumber = runtimeEntityType.AddProperty( + "SequenceNumber", + typeof(long), + propertyInfo: typeof(CheckpointMutationRow).GetProperty("SequenceNumber", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointMutationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + afterSaveBehavior: PropertySaveBehavior.Throw, + sentinel: 0L); + sequenceNumber.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + sequenceNumber.AddAnnotation("Relational:ColumnName", "sequence_number"); + + var artifactHash = runtimeEntityType.AddProperty( + "ArtifactHash", + typeof(string), + propertyInfo: typeof(CheckpointMutationRow).GetProperty("ArtifactHash", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointMutationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + artifactHash.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + artifactHash.AddAnnotation("Relational:ColumnName", "artifact_hash"); + + var artifactKind = runtimeEntityType.AddProperty( + "ArtifactKind", + typeof(string), + propertyInfo: typeof(CheckpointMutationRow).GetProperty("ArtifactKind", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointMutationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + artifactKind.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + artifactKind.AddAnnotation("Relational:ColumnName", "artifact_kind"); + + var claimsGenerated = runtimeEntityType.AddProperty( + "ClaimsGenerated", + typeof(int?), + propertyInfo: typeof(CheckpointMutationRow).GetProperty("ClaimsGenerated", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointMutationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + claimsGenerated.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + claimsGenerated.AddAnnotation("Relational:ColumnName", "claims_generated"); + + var connectorId = runtimeEntityType.AddProperty( + "ConnectorId", + typeof(string), + propertyInfo: typeof(CheckpointMutationRow).GetProperty("ConnectorId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointMutationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + connectorId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + connectorId.AddAnnotation("Relational:ColumnName", "connector_id"); + + var createdAt = runtimeEntityType.AddProperty( + "CreatedAt", + typeof(DateTime), + propertyInfo: typeof(CheckpointMutationRow).GetProperty("CreatedAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointMutationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + createdAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + createdAt.AddAnnotation("Relational:ColumnName", "created_at"); + createdAt.AddAnnotation("Relational:DefaultValueSql", "now()"); + + var cursor = runtimeEntityType.AddProperty( + "Cursor", + typeof(string), + propertyInfo: typeof(CheckpointMutationRow).GetProperty("Cursor", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointMutationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + cursor.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + cursor.AddAnnotation("Relational:ColumnName", "cursor"); + + var documentsProcessed = runtimeEntityType.AddProperty( + "DocumentsProcessed", + typeof(int?), + propertyInfo: typeof(CheckpointMutationRow).GetProperty("DocumentsProcessed", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointMutationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + documentsProcessed.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + documentsProcessed.AddAnnotation("Relational:ColumnName", "documents_processed"); + + var errorCode = runtimeEntityType.AddProperty( + "ErrorCode", + typeof(string), + propertyInfo: typeof(CheckpointMutationRow).GetProperty("ErrorCode", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointMutationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + errorCode.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + errorCode.AddAnnotation("Relational:ColumnName", "error_code"); + + var errorMessage = runtimeEntityType.AddProperty( + "ErrorMessage", + typeof(string), + propertyInfo: typeof(CheckpointMutationRow).GetProperty("ErrorMessage", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointMutationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + errorMessage.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + errorMessage.AddAnnotation("Relational:ColumnName", "error_message"); + + var idempotencyKey = runtimeEntityType.AddProperty( + "IdempotencyKey", + typeof(string), + propertyInfo: typeof(CheckpointMutationRow).GetProperty("IdempotencyKey", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointMutationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + idempotencyKey.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + idempotencyKey.AddAnnotation("Relational:ColumnName", "idempotency_key"); + + var mutationType = runtimeEntityType.AddProperty( + "MutationType", + typeof(string), + propertyInfo: typeof(CheckpointMutationRow).GetProperty("MutationType", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointMutationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + mutationType.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + mutationType.AddAnnotation("Relational:ColumnName", "mutation_type"); + + var retryAfterSeconds = runtimeEntityType.AddProperty( + "RetryAfterSeconds", + typeof(int?), + propertyInfo: typeof(CheckpointMutationRow).GetProperty("RetryAfterSeconds", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointMutationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + retryAfterSeconds.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + retryAfterSeconds.AddAnnotation("Relational:ColumnName", "retry_after_seconds"); + + var runId = runtimeEntityType.AddProperty( + "RunId", + typeof(Guid), + propertyInfo: typeof(CheckpointMutationRow).GetProperty("RunId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointMutationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: new Guid("00000000-0000-0000-0000-000000000000")); + runId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + runId.AddAnnotation("Relational:ColumnName", "run_id"); + + var tenantId = runtimeEntityType.AddProperty( + "TenantId", + typeof(string), + propertyInfo: typeof(CheckpointMutationRow).GetProperty("TenantId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointMutationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + tenantId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + tenantId.AddAnnotation("Relational:ColumnName", "tenant_id"); + + var timestamp = runtimeEntityType.AddProperty( + "Timestamp", + typeof(DateTime), + propertyInfo: typeof(CheckpointMutationRow).GetProperty("Timestamp", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointMutationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + timestamp.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + timestamp.AddAnnotation("Relational:ColumnName", "timestamp"); + + var key = runtimeEntityType.AddKey( + new[] { sequenceNumber }); + runtimeEntityType.SetPrimaryKey(key); + key.AddAnnotation("Relational:Name", "checkpoint_mutations_pkey"); + + var index = runtimeEntityType.AddIndex( + new[] { tenantId, connectorId, sequenceNumber }); + index.AddAnnotation("Relational:Name", "idx_checkpoint_mutations_tenant_connector"); + + return runtimeEntityType; + } + + public static void CreateAnnotations(RuntimeEntityType runtimeEntityType) + { + runtimeEntityType.AddAnnotation("Relational:FunctionName", null); + runtimeEntityType.AddAnnotation("Relational:Schema", "vex"); + runtimeEntityType.AddAnnotation("Relational:SqlQuery", null); + runtimeEntityType.AddAnnotation("Relational:TableName", "checkpoint_mutations"); + runtimeEntityType.AddAnnotation("Relational:ViewName", null); + runtimeEntityType.AddAnnotation("Relational:ViewSchema", null); + + Customize(runtimeEntityType); + } + + static partial void Customize(RuntimeEntityType runtimeEntityType); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/CheckpointStateRowEntityType.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/CheckpointStateRowEntityType.cs new file mode 100644 index 000000000..dbde59ab3 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/CheckpointStateRowEntityType.cs @@ -0,0 +1,193 @@ +// +using System; +using System.Reflection; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using StellaOps.Excititor.Persistence.EfCore.Models; + +#pragma warning disable 219, 612, 618 +#nullable disable + +namespace StellaOps.Excititor.Persistence.EfCore.CompiledModels +{ + [EntityFrameworkInternal] + public partial class CheckpointStateRowEntityType + { + public static RuntimeEntityType Create(RuntimeModel model, RuntimeEntityType baseEntityType = null) + { + var runtimeEntityType = model.AddEntityType( + "StellaOps.Excititor.Persistence.EfCore.Models.CheckpointStateRow", + typeof(CheckpointStateRow), + baseEntityType, + propertyCount: 15, + keyCount: 1); + + var tenantId = runtimeEntityType.AddProperty( + "TenantId", + typeof(string), + propertyInfo: typeof(CheckpointStateRow).GetProperty("TenantId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + afterSaveBehavior: PropertySaveBehavior.Throw); + tenantId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + tenantId.AddAnnotation("Relational:ColumnName", "tenant_id"); + + var connectorId = runtimeEntityType.AddProperty( + "ConnectorId", + typeof(string), + propertyInfo: typeof(CheckpointStateRow).GetProperty("ConnectorId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + afterSaveBehavior: PropertySaveBehavior.Throw); + connectorId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + connectorId.AddAnnotation("Relational:ColumnName", "connector_id"); + + var cursor = runtimeEntityType.AddProperty( + "Cursor", + typeof(string), + propertyInfo: typeof(CheckpointStateRow).GetProperty("Cursor", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + cursor.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + cursor.AddAnnotation("Relational:ColumnName", "cursor"); + + var failureCount = runtimeEntityType.AddProperty( + "FailureCount", + typeof(int), + propertyInfo: typeof(CheckpointStateRow).GetProperty("FailureCount", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: 0); + failureCount.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + failureCount.AddAnnotation("Relational:ColumnName", "failure_count"); + failureCount.AddAnnotation("Relational:DefaultValue", 0); + + var lastArtifactHash = runtimeEntityType.AddProperty( + "LastArtifactHash", + typeof(string), + propertyInfo: typeof(CheckpointStateRow).GetProperty("LastArtifactHash", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + lastArtifactHash.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + lastArtifactHash.AddAnnotation("Relational:ColumnName", "last_artifact_hash"); + + var lastArtifactKind = runtimeEntityType.AddProperty( + "LastArtifactKind", + typeof(string), + propertyInfo: typeof(CheckpointStateRow).GetProperty("LastArtifactKind", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + lastArtifactKind.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + lastArtifactKind.AddAnnotation("Relational:ColumnName", "last_artifact_kind"); + + var lastErrorCode = runtimeEntityType.AddProperty( + "LastErrorCode", + typeof(string), + propertyInfo: typeof(CheckpointStateRow).GetProperty("LastErrorCode", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + lastErrorCode.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + lastErrorCode.AddAnnotation("Relational:ColumnName", "last_error_code"); + + var lastMutationType = runtimeEntityType.AddProperty( + "LastMutationType", + typeof(string), + propertyInfo: typeof(CheckpointStateRow).GetProperty("LastMutationType", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + lastMutationType.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + lastMutationType.AddAnnotation("Relational:ColumnName", "last_mutation_type"); + + var lastRunId = runtimeEntityType.AddProperty( + "LastRunId", + typeof(Guid?), + propertyInfo: typeof(CheckpointStateRow).GetProperty("LastRunId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + lastRunId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + lastRunId.AddAnnotation("Relational:ColumnName", "last_run_id"); + + var lastUpdated = runtimeEntityType.AddProperty( + "LastUpdated", + typeof(DateTime?), + propertyInfo: typeof(CheckpointStateRow).GetProperty("LastUpdated", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + lastUpdated.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + lastUpdated.AddAnnotation("Relational:ColumnName", "last_updated"); + + var latestSequenceNumber = runtimeEntityType.AddProperty( + "LatestSequenceNumber", + typeof(long), + propertyInfo: typeof(CheckpointStateRow).GetProperty("LatestSequenceNumber", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: 0L); + latestSequenceNumber.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + latestSequenceNumber.AddAnnotation("Relational:ColumnName", "latest_sequence_number"); + latestSequenceNumber.AddAnnotation("Relational:DefaultValue", 0L); + + var nextEligibleRun = runtimeEntityType.AddProperty( + "NextEligibleRun", + typeof(DateTime?), + propertyInfo: typeof(CheckpointStateRow).GetProperty("NextEligibleRun", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + nextEligibleRun.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + nextEligibleRun.AddAnnotation("Relational:ColumnName", "next_eligible_run"); + + var successCount = runtimeEntityType.AddProperty( + "SuccessCount", + typeof(int), + propertyInfo: typeof(CheckpointStateRow).GetProperty("SuccessCount", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: 0); + successCount.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + successCount.AddAnnotation("Relational:ColumnName", "success_count"); + successCount.AddAnnotation("Relational:DefaultValue", 0); + + var totalClaimsGenerated = runtimeEntityType.AddProperty( + "TotalClaimsGenerated", + typeof(int), + propertyInfo: typeof(CheckpointStateRow).GetProperty("TotalClaimsGenerated", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: 0); + totalClaimsGenerated.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + totalClaimsGenerated.AddAnnotation("Relational:ColumnName", "total_claims_generated"); + totalClaimsGenerated.AddAnnotation("Relational:DefaultValue", 0); + + var totalDocumentsProcessed = runtimeEntityType.AddProperty( + "TotalDocumentsProcessed", + typeof(int), + propertyInfo: typeof(CheckpointStateRow).GetProperty("TotalDocumentsProcessed", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(CheckpointStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: 0); + totalDocumentsProcessed.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + totalDocumentsProcessed.AddAnnotation("Relational:ColumnName", "total_documents_processed"); + totalDocumentsProcessed.AddAnnotation("Relational:DefaultValue", 0); + + var key = runtimeEntityType.AddKey( + new[] { tenantId, connectorId }); + runtimeEntityType.SetPrimaryKey(key); + key.AddAnnotation("Relational:Name", "checkpoint_states_pkey"); + + return runtimeEntityType; + } + + public static void CreateAnnotations(RuntimeEntityType runtimeEntityType) + { + runtimeEntityType.AddAnnotation("Relational:FunctionName", null); + runtimeEntityType.AddAnnotation("Relational:Schema", "vex"); + runtimeEntityType.AddAnnotation("Relational:SqlQuery", null); + runtimeEntityType.AddAnnotation("Relational:TableName", "checkpoint_states"); + runtimeEntityType.AddAnnotation("Relational:ViewName", null); + runtimeEntityType.AddAnnotation("Relational:ViewSchema", null); + + Customize(runtimeEntityType); + } + + static partial void Customize(RuntimeEntityType runtimeEntityType); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ConnectorStateRowEntityType.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ConnectorStateRowEntityType.cs new file mode 100644 index 000000000..94134c172 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ConnectorStateRowEntityType.cs @@ -0,0 +1,169 @@ +// +using System; +using System.Reflection; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using StellaOps.Excititor.Persistence.EfCore.Models; + +#pragma warning disable 219, 612, 618 +#nullable disable + +namespace StellaOps.Excititor.Persistence.EfCore.CompiledModels +{ + [EntityFrameworkInternal] + public partial class ConnectorStateRowEntityType + { + public static RuntimeEntityType Create(RuntimeModel model, RuntimeEntityType baseEntityType = null) + { + var runtimeEntityType = model.AddEntityType( + "StellaOps.Excititor.Persistence.EfCore.Models.ConnectorStateRow", + typeof(ConnectorStateRow), + baseEntityType, + propertyCount: 13, + keyCount: 1); + + var connectorId = runtimeEntityType.AddProperty( + "ConnectorId", + typeof(string), + propertyInfo: typeof(ConnectorStateRow).GetProperty("ConnectorId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ConnectorStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + afterSaveBehavior: PropertySaveBehavior.Throw); + connectorId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + connectorId.AddAnnotation("Relational:ColumnName", "connector_id"); + + var documentDigests = runtimeEntityType.AddProperty( + "DocumentDigests", + typeof(string[]), + propertyInfo: typeof(ConnectorStateRow).GetProperty("DocumentDigests", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ConnectorStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + var documentDigestsElementType = documentDigests.SetElementType(typeof(string)); + documentDigests.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + documentDigests.AddAnnotation("Relational:ColumnName", "document_digests"); + + var failureCount = runtimeEntityType.AddProperty( + "FailureCount", + typeof(int), + propertyInfo: typeof(ConnectorStateRow).GetProperty("FailureCount", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ConnectorStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: 0); + failureCount.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + failureCount.AddAnnotation("Relational:ColumnName", "failure_count"); + failureCount.AddAnnotation("Relational:DefaultValue", 0); + + var lastArtifactHash = runtimeEntityType.AddProperty( + "LastArtifactHash", + typeof(string), + propertyInfo: typeof(ConnectorStateRow).GetProperty("LastArtifactHash", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ConnectorStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + lastArtifactHash.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + lastArtifactHash.AddAnnotation("Relational:ColumnName", "last_artifact_hash"); + + var lastArtifactKind = runtimeEntityType.AddProperty( + "LastArtifactKind", + typeof(string), + propertyInfo: typeof(ConnectorStateRow).GetProperty("LastArtifactKind", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ConnectorStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + lastArtifactKind.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + lastArtifactKind.AddAnnotation("Relational:ColumnName", "last_artifact_kind"); + + var lastCheckpoint = runtimeEntityType.AddProperty( + "LastCheckpoint", + typeof(DateTime?), + propertyInfo: typeof(ConnectorStateRow).GetProperty("LastCheckpoint", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ConnectorStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + lastCheckpoint.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + lastCheckpoint.AddAnnotation("Relational:ColumnName", "last_checkpoint"); + + var lastFailureReason = runtimeEntityType.AddProperty( + "LastFailureReason", + typeof(string), + propertyInfo: typeof(ConnectorStateRow).GetProperty("LastFailureReason", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ConnectorStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + lastFailureReason.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + lastFailureReason.AddAnnotation("Relational:ColumnName", "last_failure_reason"); + + var lastHeartbeatAt = runtimeEntityType.AddProperty( + "LastHeartbeatAt", + typeof(DateTime?), + propertyInfo: typeof(ConnectorStateRow).GetProperty("LastHeartbeatAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ConnectorStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + lastHeartbeatAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + lastHeartbeatAt.AddAnnotation("Relational:ColumnName", "last_heartbeat_at"); + + var lastHeartbeatStatus = runtimeEntityType.AddProperty( + "LastHeartbeatStatus", + typeof(string), + propertyInfo: typeof(ConnectorStateRow).GetProperty("LastHeartbeatStatus", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ConnectorStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + lastHeartbeatStatus.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + lastHeartbeatStatus.AddAnnotation("Relational:ColumnName", "last_heartbeat_status"); + + var lastSuccessAt = runtimeEntityType.AddProperty( + "LastSuccessAt", + typeof(DateTime?), + propertyInfo: typeof(ConnectorStateRow).GetProperty("LastSuccessAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ConnectorStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + lastSuccessAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + lastSuccessAt.AddAnnotation("Relational:ColumnName", "last_success_at"); + + var lastUpdated = runtimeEntityType.AddProperty( + "LastUpdated", + typeof(DateTime), + propertyInfo: typeof(ConnectorStateRow).GetProperty("LastUpdated", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ConnectorStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + lastUpdated.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + lastUpdated.AddAnnotation("Relational:ColumnName", "last_updated"); + + var nextEligibleRun = runtimeEntityType.AddProperty( + "NextEligibleRun", + typeof(DateTime?), + propertyInfo: typeof(ConnectorStateRow).GetProperty("NextEligibleRun", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ConnectorStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + nextEligibleRun.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + nextEligibleRun.AddAnnotation("Relational:ColumnName", "next_eligible_run"); + + var resumeTokens = runtimeEntityType.AddProperty( + "ResumeTokens", + typeof(string), + propertyInfo: typeof(ConnectorStateRow).GetProperty("ResumeTokens", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ConnectorStateRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd); + resumeTokens.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + resumeTokens.AddAnnotation("Relational:ColumnName", "resume_tokens"); + resumeTokens.AddAnnotation("Relational:ColumnType", "jsonb"); + resumeTokens.AddAnnotation("Relational:DefaultValueSql", "'{}'::jsonb"); + + var key = runtimeEntityType.AddKey( + new[] { connectorId }); + runtimeEntityType.SetPrimaryKey(key); + key.AddAnnotation("Relational:Name", "connector_states_pkey"); + + return runtimeEntityType; + } + + public static void CreateAnnotations(RuntimeEntityType runtimeEntityType) + { + runtimeEntityType.AddAnnotation("Relational:FunctionName", null); + runtimeEntityType.AddAnnotation("Relational:Schema", "vex"); + runtimeEntityType.AddAnnotation("Relational:SqlQuery", null); + runtimeEntityType.AddAnnotation("Relational:TableName", "connector_states"); + runtimeEntityType.AddAnnotation("Relational:ViewName", null); + runtimeEntityType.AddAnnotation("Relational:ViewSchema", null); + + Customize(runtimeEntityType); + } + + static partial void Customize(RuntimeEntityType runtimeEntityType); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/DeltaRowEntityType.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/DeltaRowEntityType.cs new file mode 100644 index 000000000..c0eca1d11 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/DeltaRowEntityType.cs @@ -0,0 +1,153 @@ +// +using System; +using System.Reflection; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using StellaOps.Excititor.Persistence.EfCore.Models; + +#pragma warning disable 219, 612, 618 +#nullable disable + +namespace StellaOps.Excititor.Persistence.EfCore.CompiledModels +{ + [EntityFrameworkInternal] + public partial class DeltaRowEntityType + { + public static RuntimeEntityType Create(RuntimeModel model, RuntimeEntityType baseEntityType = null) + { + var runtimeEntityType = model.AddEntityType( + "StellaOps.Excititor.Persistence.EfCore.Models.DeltaRow", + typeof(DeltaRow), + baseEntityType, + propertyCount: 11, + unnamedIndexCount: 1, + keyCount: 1); + + var id = runtimeEntityType.AddProperty( + "Id", + typeof(Guid), + propertyInfo: typeof(DeltaRow).GetProperty("Id", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(DeltaRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + afterSaveBehavior: PropertySaveBehavior.Throw, + sentinel: new Guid("00000000-0000-0000-0000-000000000000")); + id.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + id.AddAnnotation("Relational:ColumnName", "id"); + id.AddAnnotation("Relational:DefaultValueSql", "gen_random_uuid()"); + + var attestationDigest = runtimeEntityType.AddProperty( + "AttestationDigest", + typeof(string), + propertyInfo: typeof(DeltaRow).GetProperty("AttestationDigest", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(DeltaRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + attestationDigest.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + attestationDigest.AddAnnotation("Relational:ColumnName", "attestation_digest"); + + var createdAt = runtimeEntityType.AddProperty( + "CreatedAt", + typeof(DateTime), + propertyInfo: typeof(DeltaRow).GetProperty("CreatedAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(DeltaRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + createdAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + createdAt.AddAnnotation("Relational:ColumnName", "created_at"); + createdAt.AddAnnotation("Relational:DefaultValueSql", "now()"); + + var cve = runtimeEntityType.AddProperty( + "Cve", + typeof(string), + propertyInfo: typeof(DeltaRow).GetProperty("Cve", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(DeltaRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + cve.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + cve.AddAnnotation("Relational:ColumnName", "cve"); + + var fromArtifactDigest = runtimeEntityType.AddProperty( + "FromArtifactDigest", + typeof(string), + propertyInfo: typeof(DeltaRow).GetProperty("FromArtifactDigest", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(DeltaRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + fromArtifactDigest.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + fromArtifactDigest.AddAnnotation("Relational:ColumnName", "from_artifact_digest"); + + var fromStatus = runtimeEntityType.AddProperty( + "FromStatus", + typeof(string), + propertyInfo: typeof(DeltaRow).GetProperty("FromStatus", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(DeltaRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + fromStatus.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + fromStatus.AddAnnotation("Relational:ColumnName", "from_status"); + + var rationale = runtimeEntityType.AddProperty( + "Rationale", + typeof(string), + propertyInfo: typeof(DeltaRow).GetProperty("Rationale", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(DeltaRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + rationale.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + rationale.AddAnnotation("Relational:ColumnName", "rationale"); + rationale.AddAnnotation("Relational:ColumnType", "jsonb"); + + var replayHash = runtimeEntityType.AddProperty( + "ReplayHash", + typeof(string), + propertyInfo: typeof(DeltaRow).GetProperty("ReplayHash", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(DeltaRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + replayHash.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + replayHash.AddAnnotation("Relational:ColumnName", "replay_hash"); + + var tenantId = runtimeEntityType.AddProperty( + "TenantId", + typeof(string), + propertyInfo: typeof(DeltaRow).GetProperty("TenantId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(DeltaRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + tenantId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + tenantId.AddAnnotation("Relational:ColumnName", "tenant_id"); + + var toArtifactDigest = runtimeEntityType.AddProperty( + "ToArtifactDigest", + typeof(string), + propertyInfo: typeof(DeltaRow).GetProperty("ToArtifactDigest", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(DeltaRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + toArtifactDigest.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + toArtifactDigest.AddAnnotation("Relational:ColumnName", "to_artifact_digest"); + + var toStatus = runtimeEntityType.AddProperty( + "ToStatus", + typeof(string), + propertyInfo: typeof(DeltaRow).GetProperty("ToStatus", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(DeltaRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + toStatus.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + toStatus.AddAnnotation("Relational:ColumnName", "to_status"); + + var key = runtimeEntityType.AddKey( + new[] { id }); + runtimeEntityType.SetPrimaryKey(key); + key.AddAnnotation("Relational:Name", "deltas_pkey"); + + var index = runtimeEntityType.AddIndex( + new[] { fromArtifactDigest, toArtifactDigest, cve, tenantId }, + unique: true); + index.AddAnnotation("Relational:Name", "uq_vex_delta"); + + return runtimeEntityType; + } + + public static void CreateAnnotations(RuntimeEntityType runtimeEntityType) + { + runtimeEntityType.AddAnnotation("Relational:FunctionName", null); + runtimeEntityType.AddAnnotation("Relational:Schema", "vex"); + runtimeEntityType.AddAnnotation("Relational:SqlQuery", null); + runtimeEntityType.AddAnnotation("Relational:TableName", "deltas"); + runtimeEntityType.AddAnnotation("Relational:ViewName", null); + runtimeEntityType.AddAnnotation("Relational:ViewSchema", null); + + Customize(runtimeEntityType); + } + + static partial void Customize(RuntimeEntityType runtimeEntityType); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/EvidenceLinkEntityType.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/EvidenceLinkEntityType.cs new file mode 100644 index 000000000..4a91e093e --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/EvidenceLinkEntityType.cs @@ -0,0 +1,181 @@ +// +using System; +using System.Reflection; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using StellaOps.Excititor.Persistence.EfCore.Models; + +#pragma warning disable 219, 612, 618 +#nullable disable + +namespace StellaOps.Excititor.Persistence.EfCore.CompiledModels +{ + [EntityFrameworkInternal] + public partial class EvidenceLinkEntityType + { + public static RuntimeEntityType Create(RuntimeModel model, RuntimeEntityType baseEntityType = null) + { + var runtimeEntityType = model.AddEntityType( + "StellaOps.Excititor.Persistence.EfCore.Models.EvidenceLink", + typeof(EvidenceLink), + baseEntityType, + propertyCount: 14, + unnamedIndexCount: 2, + keyCount: 1); + + var linkId = runtimeEntityType.AddProperty( + "LinkId", + typeof(string), + propertyInfo: typeof(EvidenceLink).GetProperty("LinkId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(EvidenceLink).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + afterSaveBehavior: PropertySaveBehavior.Throw); + linkId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + linkId.AddAnnotation("Relational:ColumnName", "link_id"); + + var confidence = runtimeEntityType.AddProperty( + "Confidence", + typeof(double), + propertyInfo: typeof(EvidenceLink).GetProperty("Confidence", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(EvidenceLink).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: 0.0); + confidence.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + confidence.AddAnnotation("Relational:ColumnName", "confidence"); + + var envelopeDigest = runtimeEntityType.AddProperty( + "EnvelopeDigest", + typeof(string), + propertyInfo: typeof(EvidenceLink).GetProperty("EnvelopeDigest", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(EvidenceLink).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + envelopeDigest.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + envelopeDigest.AddAnnotation("Relational:ColumnName", "envelope_digest"); + + var evidenceCreatedAt = runtimeEntityType.AddProperty( + "EvidenceCreatedAt", + typeof(DateTime), + propertyInfo: typeof(EvidenceLink).GetProperty("EvidenceCreatedAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(EvidenceLink).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + evidenceCreatedAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + evidenceCreatedAt.AddAnnotation("Relational:ColumnName", "evidence_created_at"); + + var evidenceType = runtimeEntityType.AddProperty( + "EvidenceType", + typeof(string), + propertyInfo: typeof(EvidenceLink).GetProperty("EvidenceType", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(EvidenceLink).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + evidenceType.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + evidenceType.AddAnnotation("Relational:ColumnName", "evidence_type"); + + var evidenceUri = runtimeEntityType.AddProperty( + "EvidenceUri", + typeof(string), + propertyInfo: typeof(EvidenceLink).GetProperty("EvidenceUri", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(EvidenceLink).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + evidenceUri.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + evidenceUri.AddAnnotation("Relational:ColumnName", "evidence_uri"); + + var justification = runtimeEntityType.AddProperty( + "Justification", + typeof(string), + propertyInfo: typeof(EvidenceLink).GetProperty("Justification", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(EvidenceLink).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + justification.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + justification.AddAnnotation("Relational:ColumnName", "justification"); + + var linkedAt = runtimeEntityType.AddProperty( + "LinkedAt", + typeof(DateTime), + propertyInfo: typeof(EvidenceLink).GetProperty("LinkedAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(EvidenceLink).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + linkedAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + linkedAt.AddAnnotation("Relational:ColumnName", "linked_at"); + + var metadata = runtimeEntityType.AddProperty( + "Metadata", + typeof(string), + propertyInfo: typeof(EvidenceLink).GetProperty("Metadata", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(EvidenceLink).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd); + metadata.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + metadata.AddAnnotation("Relational:ColumnName", "metadata"); + metadata.AddAnnotation("Relational:ColumnType", "jsonb"); + metadata.AddAnnotation("Relational:DefaultValueSql", "'{}'::jsonb"); + + var predicateType = runtimeEntityType.AddProperty( + "PredicateType", + typeof(string), + propertyInfo: typeof(EvidenceLink).GetProperty("PredicateType", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(EvidenceLink).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + predicateType.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + predicateType.AddAnnotation("Relational:ColumnName", "predicate_type"); + + var rekorLogIndex = runtimeEntityType.AddProperty( + "RekorLogIndex", + typeof(string), + propertyInfo: typeof(EvidenceLink).GetProperty("RekorLogIndex", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(EvidenceLink).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + rekorLogIndex.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + rekorLogIndex.AddAnnotation("Relational:ColumnName", "rekor_log_index"); + + var signatureValidated = runtimeEntityType.AddProperty( + "SignatureValidated", + typeof(bool), + propertyInfo: typeof(EvidenceLink).GetProperty("SignatureValidated", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(EvidenceLink).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: false); + signatureValidated.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + signatureValidated.AddAnnotation("Relational:ColumnName", "signature_validated"); + signatureValidated.AddAnnotation("Relational:DefaultValue", false); + + var signerIdentity = runtimeEntityType.AddProperty( + "SignerIdentity", + typeof(string), + propertyInfo: typeof(EvidenceLink).GetProperty("SignerIdentity", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(EvidenceLink).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + signerIdentity.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + signerIdentity.AddAnnotation("Relational:ColumnName", "signer_identity"); + + var vexEntryId = runtimeEntityType.AddProperty( + "VexEntryId", + typeof(string), + propertyInfo: typeof(EvidenceLink).GetProperty("VexEntryId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(EvidenceLink).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + vexEntryId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + vexEntryId.AddAnnotation("Relational:ColumnName", "vex_entry_id"); + + var key = runtimeEntityType.AddKey( + new[] { linkId }); + runtimeEntityType.SetPrimaryKey(key); + key.AddAnnotation("Relational:Name", "evidence_links_pkey"); + + var index = runtimeEntityType.AddIndex( + new[] { envelopeDigest }); + index.AddAnnotation("Relational:Name", "ix_evidence_links_envelope_digest"); + + var index0 = runtimeEntityType.AddIndex( + new[] { vexEntryId }); + index0.AddAnnotation("Relational:Name", "ix_evidence_links_vex_entry_id"); + + return runtimeEntityType; + } + + public static void CreateAnnotations(RuntimeEntityType runtimeEntityType) + { + runtimeEntityType.AddAnnotation("Relational:FunctionName", null); + runtimeEntityType.AddAnnotation("Relational:Schema", "vex"); + runtimeEntityType.AddAnnotation("Relational:SqlQuery", null); + runtimeEntityType.AddAnnotation("Relational:TableName", "evidence_links"); + runtimeEntityType.AddAnnotation("Relational:ViewName", null); + runtimeEntityType.AddAnnotation("Relational:ViewSchema", null); + + Customize(runtimeEntityType); + } + + static partial void Customize(RuntimeEntityType runtimeEntityType); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ExcititorDbContextAssemblyAttributes.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ExcititorDbContextAssemblyAttributes.cs new file mode 100644 index 000000000..3cbfc3ab1 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ExcititorDbContextAssemblyAttributes.cs @@ -0,0 +1,9 @@ +// +using Microsoft.EntityFrameworkCore.Infrastructure; +using StellaOps.Excititor.Persistence.EfCore.CompiledModels; +using StellaOps.Excititor.Persistence.EfCore.Context; + +#pragma warning disable 219, 612, 618 +#nullable disable + +[assembly: DbContextModel(typeof(ExcititorDbContext), typeof(ExcititorDbContextModel))] diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ExcititorDbContextModel.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ExcititorDbContextModel.cs index 4aea8458b..b13c65042 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ExcititorDbContextModel.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ExcititorDbContextModel.cs @@ -1,4 +1,4 @@ -// +// using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Metadata; using StellaOps.Excititor.Persistence.EfCore.Context; diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ExcititorDbContextModelBuilder.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ExcititorDbContextModelBuilder.cs new file mode 100644 index 000000000..02d317e10 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ExcititorDbContextModelBuilder.cs @@ -0,0 +1,66 @@ +// +using System; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#pragma warning disable 219, 612, 618 +#nullable disable + +namespace StellaOps.Excititor.Persistence.EfCore.CompiledModels +{ + public partial class ExcititorDbContextModel + { + private ExcititorDbContextModel() + : base(skipDetectChanges: false, modelId: new Guid("2ac0e4a0-f551-47b6-b2d1-e91959b18214"), entityTypeCount: 19) + { + } + + partial void Initialize() + { + var attestationRow = AttestationRowEntityType.Create(this); + var calibrationAdjustment = CalibrationAdjustmentEntityType.Create(this); + var calibrationManifest = CalibrationManifestEntityType.Create(this); + var checkpointMutationRow = CheckpointMutationRowEntityType.Create(this); + var checkpointStateRow = CheckpointStateRowEntityType.Create(this); + var connectorStateRow = ConnectorStateRowEntityType.Create(this); + var deltaRow = DeltaRowEntityType.Create(this); + var evidenceLink = EvidenceLinkEntityType.Create(this); + var linkset = LinksetEntityType.Create(this); + var linksetDisagreement = LinksetDisagreementEntityType.Create(this); + var linksetMutation = LinksetMutationEntityType.Create(this); + var linksetObservation = LinksetObservationEntityType.Create(this); + var observationRow = ObservationRowEntityType.Create(this); + var observationTimelineEventRow = ObservationTimelineEventRowEntityType.Create(this); + var providerRow = ProviderRowEntityType.Create(this); + var sourceTrustVector = SourceTrustVectorEntityType.Create(this); + var statementRow = StatementRowEntityType.Create(this); + var vexRawBlob = VexRawBlobEntityType.Create(this); + var vexRawDocument = VexRawDocumentEntityType.Create(this); + + AttestationRowEntityType.CreateAnnotations(attestationRow); + CalibrationAdjustmentEntityType.CreateAnnotations(calibrationAdjustment); + CalibrationManifestEntityType.CreateAnnotations(calibrationManifest); + CheckpointMutationRowEntityType.CreateAnnotations(checkpointMutationRow); + CheckpointStateRowEntityType.CreateAnnotations(checkpointStateRow); + ConnectorStateRowEntityType.CreateAnnotations(connectorStateRow); + DeltaRowEntityType.CreateAnnotations(deltaRow); + EvidenceLinkEntityType.CreateAnnotations(evidenceLink); + LinksetEntityType.CreateAnnotations(linkset); + LinksetDisagreementEntityType.CreateAnnotations(linksetDisagreement); + LinksetMutationEntityType.CreateAnnotations(linksetMutation); + LinksetObservationEntityType.CreateAnnotations(linksetObservation); + ObservationRowEntityType.CreateAnnotations(observationRow); + ObservationTimelineEventRowEntityType.CreateAnnotations(observationTimelineEventRow); + ProviderRowEntityType.CreateAnnotations(providerRow); + SourceTrustVectorEntityType.CreateAnnotations(sourceTrustVector); + StatementRowEntityType.CreateAnnotations(statementRow); + VexRawBlobEntityType.CreateAnnotations(vexRawBlob); + VexRawDocumentEntityType.CreateAnnotations(vexRawDocument); + + AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + AddAnnotation("ProductVersion", "10.0.0"); + AddAnnotation("Relational:MaxIdentifierLength", 63); + } + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/LinksetDisagreementEntityType.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/LinksetDisagreementEntityType.cs new file mode 100644 index 000000000..58e19aedf --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/LinksetDisagreementEntityType.cs @@ -0,0 +1,123 @@ +// +using System; +using System.Reflection; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using StellaOps.Excititor.Persistence.EfCore.Models; + +#pragma warning disable 219, 612, 618 +#nullable disable + +namespace StellaOps.Excititor.Persistence.EfCore.CompiledModels +{ + [EntityFrameworkInternal] + public partial class LinksetDisagreementEntityType + { + public static RuntimeEntityType Create(RuntimeModel model, RuntimeEntityType baseEntityType = null) + { + var runtimeEntityType = model.AddEntityType( + "StellaOps.Excititor.Persistence.EfCore.Models.LinksetDisagreement", + typeof(LinksetDisagreement), + baseEntityType, + propertyCount: 7, + unnamedIndexCount: 2, + keyCount: 1); + + var id = runtimeEntityType.AddProperty( + "Id", + typeof(long), + propertyInfo: typeof(LinksetDisagreement).GetProperty("Id", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetDisagreement).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + afterSaveBehavior: PropertySaveBehavior.Throw, + sentinel: 0L); + id.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + id.AddAnnotation("Relational:ColumnName", "id"); + + var confidence = runtimeEntityType.AddProperty( + "Confidence", + typeof(decimal?), + propertyInfo: typeof(LinksetDisagreement).GetProperty("Confidence", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetDisagreement).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + confidence.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + confidence.AddAnnotation("Relational:ColumnName", "confidence"); + confidence.AddAnnotation("Relational:ColumnType", "numeric(4,3)"); + + var createdAt = runtimeEntityType.AddProperty( + "CreatedAt", + typeof(DateTime), + propertyInfo: typeof(LinksetDisagreement).GetProperty("CreatedAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetDisagreement).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + createdAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + createdAt.AddAnnotation("Relational:ColumnName", "created_at"); + createdAt.AddAnnotation("Relational:DefaultValueSql", "now()"); + + var justification = runtimeEntityType.AddProperty( + "Justification", + typeof(string), + propertyInfo: typeof(LinksetDisagreement).GetProperty("Justification", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetDisagreement).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + justification.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + justification.AddAnnotation("Relational:ColumnName", "justification"); + + var linksetId = runtimeEntityType.AddProperty( + "LinksetId", + typeof(string), + propertyInfo: typeof(LinksetDisagreement).GetProperty("LinksetId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetDisagreement).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + linksetId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + linksetId.AddAnnotation("Relational:ColumnName", "linkset_id"); + + var providerId = runtimeEntityType.AddProperty( + "ProviderId", + typeof(string), + propertyInfo: typeof(LinksetDisagreement).GetProperty("ProviderId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetDisagreement).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + providerId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + providerId.AddAnnotation("Relational:ColumnName", "provider_id"); + + var status = runtimeEntityType.AddProperty( + "Status", + typeof(string), + propertyInfo: typeof(LinksetDisagreement).GetProperty("Status", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetDisagreement).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + status.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + status.AddAnnotation("Relational:ColumnName", "status"); + + var key = runtimeEntityType.AddKey( + new[] { id }); + runtimeEntityType.SetPrimaryKey(key); + key.AddAnnotation("Relational:Name", "linkset_disagreements_pkey"); + + var index = runtimeEntityType.AddIndex( + new[] { linksetId }); + index.AddAnnotation("Relational:Name", "idx_linkset_disagreements_linkset"); + + var index0 = runtimeEntityType.AddIndex( + new[] { linksetId, providerId, status, justification }, + unique: true); + index0.AddAnnotation("Relational:Name", "linkset_disagreements_linkset_id_provider_id_status_justificat"); + + return runtimeEntityType; + } + + public static void CreateAnnotations(RuntimeEntityType runtimeEntityType) + { + runtimeEntityType.AddAnnotation("Relational:FunctionName", null); + runtimeEntityType.AddAnnotation("Relational:Schema", "vex"); + runtimeEntityType.AddAnnotation("Relational:SqlQuery", null); + runtimeEntityType.AddAnnotation("Relational:TableName", "linkset_disagreements"); + runtimeEntityType.AddAnnotation("Relational:ViewName", null); + runtimeEntityType.AddAnnotation("Relational:ViewSchema", null); + + Customize(runtimeEntityType); + } + + static partial void Customize(RuntimeEntityType runtimeEntityType); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/LinksetEntityType.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/LinksetEntityType.cs new file mode 100644 index 000000000..11f13a588 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/LinksetEntityType.cs @@ -0,0 +1,122 @@ +// +using System; +using System.Reflection; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using StellaOps.Excititor.Persistence.EfCore.Models; + +#pragma warning disable 219, 612, 618 +#nullable disable + +namespace StellaOps.Excititor.Persistence.EfCore.CompiledModels +{ + [EntityFrameworkInternal] + public partial class LinksetEntityType + { + public static RuntimeEntityType Create(RuntimeModel model, RuntimeEntityType baseEntityType = null) + { + var runtimeEntityType = model.AddEntityType( + "StellaOps.Excititor.Persistence.EfCore.Models.Linkset", + typeof(Linkset), + baseEntityType, + propertyCount: 7, + unnamedIndexCount: 2, + keyCount: 1); + + var linksetId = runtimeEntityType.AddProperty( + "LinksetId", + typeof(string), + propertyInfo: typeof(Linkset).GetProperty("LinksetId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(Linkset).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + afterSaveBehavior: PropertySaveBehavior.Throw); + linksetId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + linksetId.AddAnnotation("Relational:ColumnName", "linkset_id"); + + var createdAt = runtimeEntityType.AddProperty( + "CreatedAt", + typeof(DateTime), + propertyInfo: typeof(Linkset).GetProperty("CreatedAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(Linkset).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + createdAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + createdAt.AddAnnotation("Relational:ColumnName", "created_at"); + createdAt.AddAnnotation("Relational:DefaultValueSql", "now()"); + + var productKey = runtimeEntityType.AddProperty( + "ProductKey", + typeof(string), + propertyInfo: typeof(Linkset).GetProperty("ProductKey", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(Linkset).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + productKey.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + productKey.AddAnnotation("Relational:ColumnName", "product_key"); + + var scope = runtimeEntityType.AddProperty( + "Scope", + typeof(string), + propertyInfo: typeof(Linkset).GetProperty("Scope", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(Linkset).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + scope.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + scope.AddAnnotation("Relational:ColumnName", "scope"); + scope.AddAnnotation("Relational:ColumnType", "jsonb"); + + var tenant = runtimeEntityType.AddProperty( + "Tenant", + typeof(string), + propertyInfo: typeof(Linkset).GetProperty("Tenant", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(Linkset).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + tenant.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + tenant.AddAnnotation("Relational:ColumnName", "tenant"); + + var updatedAt = runtimeEntityType.AddProperty( + "UpdatedAt", + typeof(DateTime), + propertyInfo: typeof(Linkset).GetProperty("UpdatedAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(Linkset).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + updatedAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + updatedAt.AddAnnotation("Relational:ColumnName", "updated_at"); + updatedAt.AddAnnotation("Relational:DefaultValueSql", "now()"); + + var vulnerabilityId = runtimeEntityType.AddProperty( + "VulnerabilityId", + typeof(string), + propertyInfo: typeof(Linkset).GetProperty("VulnerabilityId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(Linkset).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + vulnerabilityId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + vulnerabilityId.AddAnnotation("Relational:ColumnName", "vulnerability_id"); + + var key = runtimeEntityType.AddKey( + new[] { linksetId }); + runtimeEntityType.SetPrimaryKey(key); + key.AddAnnotation("Relational:Name", "linksets_pkey"); + + var index = runtimeEntityType.AddIndex( + new[] { tenant, updatedAt }); + index.AddAnnotation("Relational:Name", "idx_linksets_updated"); + + var index0 = runtimeEntityType.AddIndex( + new[] { tenant, vulnerabilityId, productKey }, + unique: true); + index0.AddAnnotation("Relational:Name", "linksets_tenant_vulnerability_id_product_key_key"); + + return runtimeEntityType; + } + + public static void CreateAnnotations(RuntimeEntityType runtimeEntityType) + { + runtimeEntityType.AddAnnotation("Relational:FunctionName", null); + runtimeEntityType.AddAnnotation("Relational:Schema", "vex"); + runtimeEntityType.AddAnnotation("Relational:SqlQuery", null); + runtimeEntityType.AddAnnotation("Relational:TableName", "linksets"); + runtimeEntityType.AddAnnotation("Relational:ViewName", null); + runtimeEntityType.AddAnnotation("Relational:ViewSchema", null); + + Customize(runtimeEntityType); + } + + static partial void Customize(RuntimeEntityType runtimeEntityType); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/LinksetMutationEntityType.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/LinksetMutationEntityType.cs new file mode 100644 index 000000000..b720560d7 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/LinksetMutationEntityType.cs @@ -0,0 +1,137 @@ +// +using System; +using System.Reflection; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using StellaOps.Excititor.Persistence.EfCore.Models; + +#pragma warning disable 219, 612, 618 +#nullable disable + +namespace StellaOps.Excititor.Persistence.EfCore.CompiledModels +{ + [EntityFrameworkInternal] + public partial class LinksetMutationEntityType + { + public static RuntimeEntityType Create(RuntimeModel model, RuntimeEntityType baseEntityType = null) + { + var runtimeEntityType = model.AddEntityType( + "StellaOps.Excititor.Persistence.EfCore.Models.LinksetMutation", + typeof(LinksetMutation), + baseEntityType, + propertyCount: 9, + unnamedIndexCount: 1, + keyCount: 1); + + var sequenceNumber = runtimeEntityType.AddProperty( + "SequenceNumber", + typeof(long), + propertyInfo: typeof(LinksetMutation).GetProperty("SequenceNumber", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetMutation).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + afterSaveBehavior: PropertySaveBehavior.Throw, + sentinel: 0L); + sequenceNumber.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + sequenceNumber.AddAnnotation("Relational:ColumnName", "sequence_number"); + + var confidence = runtimeEntityType.AddProperty( + "Confidence", + typeof(decimal?), + propertyInfo: typeof(LinksetMutation).GetProperty("Confidence", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetMutation).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + confidence.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + confidence.AddAnnotation("Relational:ColumnName", "confidence"); + confidence.AddAnnotation("Relational:ColumnType", "numeric(4,3)"); + + var justification = runtimeEntityType.AddProperty( + "Justification", + typeof(string), + propertyInfo: typeof(LinksetMutation).GetProperty("Justification", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetMutation).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + justification.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + justification.AddAnnotation("Relational:ColumnName", "justification"); + + var linksetId = runtimeEntityType.AddProperty( + "LinksetId", + typeof(string), + propertyInfo: typeof(LinksetMutation).GetProperty("LinksetId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetMutation).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + linksetId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + linksetId.AddAnnotation("Relational:ColumnName", "linkset_id"); + + var mutationType = runtimeEntityType.AddProperty( + "MutationType", + typeof(string), + propertyInfo: typeof(LinksetMutation).GetProperty("MutationType", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetMutation).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + mutationType.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + mutationType.AddAnnotation("Relational:ColumnName", "mutation_type"); + + var observationId = runtimeEntityType.AddProperty( + "ObservationId", + typeof(string), + propertyInfo: typeof(LinksetMutation).GetProperty("ObservationId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetMutation).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + observationId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + observationId.AddAnnotation("Relational:ColumnName", "observation_id"); + + var occurredAt = runtimeEntityType.AddProperty( + "OccurredAt", + typeof(DateTime), + propertyInfo: typeof(LinksetMutation).GetProperty("OccurredAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetMutation).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + occurredAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + occurredAt.AddAnnotation("Relational:ColumnName", "occurred_at"); + occurredAt.AddAnnotation("Relational:DefaultValueSql", "now()"); + + var providerId = runtimeEntityType.AddProperty( + "ProviderId", + typeof(string), + propertyInfo: typeof(LinksetMutation).GetProperty("ProviderId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetMutation).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + providerId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + providerId.AddAnnotation("Relational:ColumnName", "provider_id"); + + var status = runtimeEntityType.AddProperty( + "Status", + typeof(string), + propertyInfo: typeof(LinksetMutation).GetProperty("Status", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetMutation).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + status.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + status.AddAnnotation("Relational:ColumnName", "status"); + + var key = runtimeEntityType.AddKey( + new[] { sequenceNumber }); + runtimeEntityType.SetPrimaryKey(key); + key.AddAnnotation("Relational:Name", "linkset_mutations_pkey"); + + var index = runtimeEntityType.AddIndex( + new[] { linksetId, sequenceNumber }); + index.AddAnnotation("Relational:Name", "idx_linkset_mutations_linkset"); + + return runtimeEntityType; + } + + public static void CreateAnnotations(RuntimeEntityType runtimeEntityType) + { + runtimeEntityType.AddAnnotation("Relational:FunctionName", null); + runtimeEntityType.AddAnnotation("Relational:Schema", "vex"); + runtimeEntityType.AddAnnotation("Relational:SqlQuery", null); + runtimeEntityType.AddAnnotation("Relational:TableName", "linkset_mutations"); + runtimeEntityType.AddAnnotation("Relational:ViewName", null); + runtimeEntityType.AddAnnotation("Relational:ViewSchema", null); + + Customize(runtimeEntityType); + } + + static partial void Customize(RuntimeEntityType runtimeEntityType); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/LinksetObservationEntityType.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/LinksetObservationEntityType.cs new file mode 100644 index 000000000..ef1ca1803 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/LinksetObservationEntityType.cs @@ -0,0 +1,130 @@ +// +using System; +using System.Reflection; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using StellaOps.Excititor.Persistence.EfCore.Models; + +#pragma warning disable 219, 612, 618 +#nullable disable + +namespace StellaOps.Excititor.Persistence.EfCore.CompiledModels +{ + [EntityFrameworkInternal] + public partial class LinksetObservationEntityType + { + public static RuntimeEntityType Create(RuntimeModel model, RuntimeEntityType baseEntityType = null) + { + var runtimeEntityType = model.AddEntityType( + "StellaOps.Excititor.Persistence.EfCore.Models.LinksetObservation", + typeof(LinksetObservation), + baseEntityType, + propertyCount: 7, + unnamedIndexCount: 4, + keyCount: 1); + + var id = runtimeEntityType.AddProperty( + "Id", + typeof(long), + propertyInfo: typeof(LinksetObservation).GetProperty("Id", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetObservation).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + afterSaveBehavior: PropertySaveBehavior.Throw, + sentinel: 0L); + id.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + id.AddAnnotation("Relational:ColumnName", "id"); + + var confidence = runtimeEntityType.AddProperty( + "Confidence", + typeof(decimal?), + propertyInfo: typeof(LinksetObservation).GetProperty("Confidence", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetObservation).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + confidence.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + confidence.AddAnnotation("Relational:ColumnName", "confidence"); + confidence.AddAnnotation("Relational:ColumnType", "numeric(4,3)"); + + var createdAt = runtimeEntityType.AddProperty( + "CreatedAt", + typeof(DateTime), + propertyInfo: typeof(LinksetObservation).GetProperty("CreatedAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetObservation).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + createdAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + createdAt.AddAnnotation("Relational:ColumnName", "created_at"); + createdAt.AddAnnotation("Relational:DefaultValueSql", "now()"); + + var linksetId = runtimeEntityType.AddProperty( + "LinksetId", + typeof(string), + propertyInfo: typeof(LinksetObservation).GetProperty("LinksetId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetObservation).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + linksetId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + linksetId.AddAnnotation("Relational:ColumnName", "linkset_id"); + + var observationId = runtimeEntityType.AddProperty( + "ObservationId", + typeof(string), + propertyInfo: typeof(LinksetObservation).GetProperty("ObservationId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetObservation).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + observationId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + observationId.AddAnnotation("Relational:ColumnName", "observation_id"); + + var providerId = runtimeEntityType.AddProperty( + "ProviderId", + typeof(string), + propertyInfo: typeof(LinksetObservation).GetProperty("ProviderId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetObservation).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + providerId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + providerId.AddAnnotation("Relational:ColumnName", "provider_id"); + + var status = runtimeEntityType.AddProperty( + "Status", + typeof(string), + propertyInfo: typeof(LinksetObservation).GetProperty("Status", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(LinksetObservation).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + status.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + status.AddAnnotation("Relational:ColumnName", "status"); + + var key = runtimeEntityType.AddKey( + new[] { id }); + runtimeEntityType.SetPrimaryKey(key); + key.AddAnnotation("Relational:Name", "linkset_observations_pkey"); + + var index = runtimeEntityType.AddIndex( + new[] { linksetId }); + index.AddAnnotation("Relational:Name", "idx_linkset_observations_linkset"); + + var index0 = runtimeEntityType.AddIndex( + new[] { linksetId, providerId }); + index0.AddAnnotation("Relational:Name", "idx_linkset_observations_provider"); + + var index1 = runtimeEntityType.AddIndex( + new[] { linksetId, status }); + index1.AddAnnotation("Relational:Name", "idx_linkset_observations_status"); + + var index2 = runtimeEntityType.AddIndex( + new[] { linksetId, observationId, providerId, status }, + unique: true); + index2.AddAnnotation("Relational:Name", "linkset_observations_linkset_id_observation_id_provider_id_sta"); + + return runtimeEntityType; + } + + public static void CreateAnnotations(RuntimeEntityType runtimeEntityType) + { + runtimeEntityType.AddAnnotation("Relational:FunctionName", null); + runtimeEntityType.AddAnnotation("Relational:Schema", "vex"); + runtimeEntityType.AddAnnotation("Relational:SqlQuery", null); + runtimeEntityType.AddAnnotation("Relational:TableName", "linkset_observations"); + runtimeEntityType.AddAnnotation("Relational:ViewName", null); + runtimeEntityType.AddAnnotation("Relational:ViewSchema", null); + + Customize(runtimeEntityType); + } + + static partial void Customize(RuntimeEntityType runtimeEntityType); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ObservationRowEntityType.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ObservationRowEntityType.cs new file mode 100644 index 000000000..936eddec9 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ObservationRowEntityType.cs @@ -0,0 +1,255 @@ +// +using System; +using System.Reflection; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using StellaOps.Excititor.Persistence.EfCore.Models; + +#pragma warning disable 219, 612, 618 +#nullable disable + +namespace StellaOps.Excititor.Persistence.EfCore.CompiledModels +{ + [EntityFrameworkInternal] + public partial class ObservationRowEntityType + { + public static RuntimeEntityType Create(RuntimeModel model, RuntimeEntityType baseEntityType = null) + { + var runtimeEntityType = model.AddEntityType( + "StellaOps.Excititor.Persistence.EfCore.Models.ObservationRow", + typeof(ObservationRow), + baseEntityType, + propertyCount: 21, + unnamedIndexCount: 3, + keyCount: 1); + + var tenant = runtimeEntityType.AddProperty( + "Tenant", + typeof(string), + propertyInfo: typeof(ObservationRow).GetProperty("Tenant", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + afterSaveBehavior: PropertySaveBehavior.Throw); + tenant.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + tenant.AddAnnotation("Relational:ColumnName", "tenant"); + + var observationId = runtimeEntityType.AddProperty( + "ObservationId", + typeof(string), + propertyInfo: typeof(ObservationRow).GetProperty("ObservationId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + afterSaveBehavior: PropertySaveBehavior.Throw); + observationId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + observationId.AddAnnotation("Relational:ColumnName", "observation_id"); + + var attributes = runtimeEntityType.AddProperty( + "Attributes", + typeof(string), + propertyInfo: typeof(ObservationRow).GetProperty("Attributes", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd); + attributes.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + attributes.AddAnnotation("Relational:ColumnName", "attributes"); + attributes.AddAnnotation("Relational:ColumnType", "jsonb"); + attributes.AddAnnotation("Relational:DefaultValueSql", "'{}'::jsonb"); + + var content = runtimeEntityType.AddProperty( + "Content", + typeof(string), + propertyInfo: typeof(ObservationRow).GetProperty("Content", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + content.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + content.AddAnnotation("Relational:ColumnName", "content"); + content.AddAnnotation("Relational:ColumnType", "jsonb"); + + var createdAt = runtimeEntityType.AddProperty( + "CreatedAt", + typeof(DateTime), + propertyInfo: typeof(ObservationRow).GetProperty("CreatedAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + createdAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + createdAt.AddAnnotation("Relational:ColumnName", "created_at"); + + var linkset = runtimeEntityType.AddProperty( + "Linkset", + typeof(string), + propertyInfo: typeof(ObservationRow).GetProperty("Linkset", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd); + linkset.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + linkset.AddAnnotation("Relational:ColumnName", "linkset"); + linkset.AddAnnotation("Relational:ColumnType", "jsonb"); + linkset.AddAnnotation("Relational:DefaultValueSql", "'{}'::jsonb"); + + var providerId = runtimeEntityType.AddProperty( + "ProviderId", + typeof(string), + propertyInfo: typeof(ObservationRow).GetProperty("ProviderId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + providerId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + providerId.AddAnnotation("Relational:ColumnName", "provider_id"); + + var rekorEntryBodyHash = runtimeEntityType.AddProperty( + "RekorEntryBodyHash", + typeof(string), + propertyInfo: typeof(ObservationRow).GetProperty("RekorEntryBodyHash", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + rekorEntryBodyHash.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + rekorEntryBodyHash.AddAnnotation("Relational:ColumnName", "rekor_entry_body_hash"); + + var rekorEntryKind = runtimeEntityType.AddProperty( + "RekorEntryKind", + typeof(string), + propertyInfo: typeof(ObservationRow).GetProperty("RekorEntryKind", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + rekorEntryKind.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + rekorEntryKind.AddAnnotation("Relational:ColumnName", "rekor_entry_kind"); + + var rekorInclusionProof = runtimeEntityType.AddProperty( + "RekorInclusionProof", + typeof(string), + propertyInfo: typeof(ObservationRow).GetProperty("RekorInclusionProof", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + rekorInclusionProof.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + rekorInclusionProof.AddAnnotation("Relational:ColumnName", "rekor_inclusion_proof"); + rekorInclusionProof.AddAnnotation("Relational:ColumnType", "jsonb"); + + var rekorIntegratedTime = runtimeEntityType.AddProperty( + "RekorIntegratedTime", + typeof(DateTime?), + propertyInfo: typeof(ObservationRow).GetProperty("RekorIntegratedTime", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + rekorIntegratedTime.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + rekorIntegratedTime.AddAnnotation("Relational:ColumnName", "rekor_integrated_time"); + + var rekorLinkedAt = runtimeEntityType.AddProperty( + "RekorLinkedAt", + typeof(DateTime?), + propertyInfo: typeof(ObservationRow).GetProperty("RekorLinkedAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + rekorLinkedAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + rekorLinkedAt.AddAnnotation("Relational:ColumnName", "rekor_linked_at"); + + var rekorLogIndex = runtimeEntityType.AddProperty( + "RekorLogIndex", + typeof(long?), + propertyInfo: typeof(ObservationRow).GetProperty("RekorLogIndex", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + rekorLogIndex.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + rekorLogIndex.AddAnnotation("Relational:ColumnName", "rekor_log_index"); + + var rekorLogUrl = runtimeEntityType.AddProperty( + "RekorLogUrl", + typeof(string), + propertyInfo: typeof(ObservationRow).GetProperty("RekorLogUrl", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + rekorLogUrl.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + rekorLogUrl.AddAnnotation("Relational:ColumnName", "rekor_log_url"); + + var rekorTreeRoot = runtimeEntityType.AddProperty( + "RekorTreeRoot", + typeof(string), + propertyInfo: typeof(ObservationRow).GetProperty("RekorTreeRoot", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + rekorTreeRoot.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + rekorTreeRoot.AddAnnotation("Relational:ColumnName", "rekor_tree_root"); + + var rekorTreeSize = runtimeEntityType.AddProperty( + "RekorTreeSize", + typeof(long?), + propertyInfo: typeof(ObservationRow).GetProperty("RekorTreeSize", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + rekorTreeSize.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + rekorTreeSize.AddAnnotation("Relational:ColumnName", "rekor_tree_size"); + + var rekorUuid = runtimeEntityType.AddProperty( + "RekorUuid", + typeof(string), + propertyInfo: typeof(ObservationRow).GetProperty("RekorUuid", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + rekorUuid.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + rekorUuid.AddAnnotation("Relational:ColumnName", "rekor_uuid"); + + var statements = runtimeEntityType.AddProperty( + "Statements", + typeof(string), + propertyInfo: typeof(ObservationRow).GetProperty("Statements", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd); + statements.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + statements.AddAnnotation("Relational:ColumnName", "statements"); + statements.AddAnnotation("Relational:ColumnType", "jsonb"); + statements.AddAnnotation("Relational:DefaultValueSql", "'[]'::jsonb"); + + var streamId = runtimeEntityType.AddProperty( + "StreamId", + typeof(string), + propertyInfo: typeof(ObservationRow).GetProperty("StreamId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + streamId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + streamId.AddAnnotation("Relational:ColumnName", "stream_id"); + + var supersedes = runtimeEntityType.AddProperty( + "Supersedes", + typeof(string[]), + propertyInfo: typeof(ObservationRow).GetProperty("Supersedes", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + var supersedesElementType = supersedes.SetElementType(typeof(string)); + supersedes.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + supersedes.AddAnnotation("Relational:ColumnName", "supersedes"); + + var upstream = runtimeEntityType.AddProperty( + "Upstream", + typeof(string), + propertyInfo: typeof(ObservationRow).GetProperty("Upstream", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + upstream.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + upstream.AddAnnotation("Relational:ColumnName", "upstream"); + upstream.AddAnnotation("Relational:ColumnType", "jsonb"); + + var key = runtimeEntityType.AddKey( + new[] { tenant, observationId }); + runtimeEntityType.SetPrimaryKey(key); + key.AddAnnotation("Relational:Name", "observations_pkey"); + + var index = runtimeEntityType.AddIndex( + new[] { tenant }); + index.AddAnnotation("Relational:Name", "idx_observations_tenant"); + + var index0 = runtimeEntityType.AddIndex( + new[] { tenant, createdAt }); + index0.AddAnnotation("Relational:Name", "idx_observations_created_at"); + + var index1 = runtimeEntityType.AddIndex( + new[] { tenant, providerId }); + index1.AddAnnotation("Relational:Name", "idx_observations_provider"); + + return runtimeEntityType; + } + + public static void CreateAnnotations(RuntimeEntityType runtimeEntityType) + { + runtimeEntityType.AddAnnotation("Relational:FunctionName", null); + runtimeEntityType.AddAnnotation("Relational:Schema", "vex"); + runtimeEntityType.AddAnnotation("Relational:SqlQuery", null); + runtimeEntityType.AddAnnotation("Relational:TableName", "observations"); + runtimeEntityType.AddAnnotation("Relational:ViewName", null); + runtimeEntityType.AddAnnotation("Relational:ViewSchema", null); + + Customize(runtimeEntityType); + } + + static partial void Customize(RuntimeEntityType runtimeEntityType); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ObservationTimelineEventRowEntityType.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ObservationTimelineEventRowEntityType.cs new file mode 100644 index 000000000..cd59b4e9d --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ObservationTimelineEventRowEntityType.cs @@ -0,0 +1,167 @@ +// +using System; +using System.Reflection; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using StellaOps.Excititor.Persistence.EfCore.Models; + +#pragma warning disable 219, 612, 618 +#nullable disable + +namespace StellaOps.Excititor.Persistence.EfCore.CompiledModels +{ + [EntityFrameworkInternal] + public partial class ObservationTimelineEventRowEntityType + { + public static RuntimeEntityType Create(RuntimeModel model, RuntimeEntityType baseEntityType = null) + { + var runtimeEntityType = model.AddEntityType( + "StellaOps.Excititor.Persistence.EfCore.Models.ObservationTimelineEventRow", + typeof(ObservationTimelineEventRow), + baseEntityType, + propertyCount: 11, + unnamedIndexCount: 5, + keyCount: 1); + + var tenant = runtimeEntityType.AddProperty( + "Tenant", + typeof(string), + propertyInfo: typeof(ObservationTimelineEventRow).GetProperty("Tenant", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationTimelineEventRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + afterSaveBehavior: PropertySaveBehavior.Throw); + tenant.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + tenant.AddAnnotation("Relational:ColumnName", "tenant"); + + var eventId = runtimeEntityType.AddProperty( + "EventId", + typeof(string), + propertyInfo: typeof(ObservationTimelineEventRow).GetProperty("EventId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationTimelineEventRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + afterSaveBehavior: PropertySaveBehavior.Throw); + eventId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + eventId.AddAnnotation("Relational:ColumnName", "event_id"); + + var attributes = runtimeEntityType.AddProperty( + "Attributes", + typeof(string), + propertyInfo: typeof(ObservationTimelineEventRow).GetProperty("Attributes", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationTimelineEventRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd); + attributes.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + attributes.AddAnnotation("Relational:ColumnName", "attributes"); + attributes.AddAnnotation("Relational:ColumnType", "jsonb"); + attributes.AddAnnotation("Relational:DefaultValueSql", "'{}'::jsonb"); + + var createdAt = runtimeEntityType.AddProperty( + "CreatedAt", + typeof(DateTime), + propertyInfo: typeof(ObservationTimelineEventRow).GetProperty("CreatedAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationTimelineEventRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + createdAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + createdAt.AddAnnotation("Relational:ColumnName", "created_at"); + + var eventType = runtimeEntityType.AddProperty( + "EventType", + typeof(string), + propertyInfo: typeof(ObservationTimelineEventRow).GetProperty("EventType", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationTimelineEventRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + eventType.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + eventType.AddAnnotation("Relational:ColumnName", "event_type"); + + var evidenceHash = runtimeEntityType.AddProperty( + "EvidenceHash", + typeof(string), + propertyInfo: typeof(ObservationTimelineEventRow).GetProperty("EvidenceHash", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationTimelineEventRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + evidenceHash.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + evidenceHash.AddAnnotation("Relational:ColumnName", "evidence_hash"); + + var justificationSummary = runtimeEntityType.AddProperty( + "JustificationSummary", + typeof(string), + propertyInfo: typeof(ObservationTimelineEventRow).GetProperty("JustificationSummary", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationTimelineEventRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd); + justificationSummary.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + justificationSummary.AddAnnotation("Relational:ColumnName", "justification_summary"); + justificationSummary.AddAnnotation("Relational:DefaultValue", ""); + + var payloadHash = runtimeEntityType.AddProperty( + "PayloadHash", + typeof(string), + propertyInfo: typeof(ObservationTimelineEventRow).GetProperty("PayloadHash", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationTimelineEventRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + payloadHash.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + payloadHash.AddAnnotation("Relational:ColumnName", "payload_hash"); + + var providerId = runtimeEntityType.AddProperty( + "ProviderId", + typeof(string), + propertyInfo: typeof(ObservationTimelineEventRow).GetProperty("ProviderId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationTimelineEventRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + providerId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + providerId.AddAnnotation("Relational:ColumnName", "provider_id"); + + var streamId = runtimeEntityType.AddProperty( + "StreamId", + typeof(string), + propertyInfo: typeof(ObservationTimelineEventRow).GetProperty("StreamId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationTimelineEventRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + streamId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + streamId.AddAnnotation("Relational:ColumnName", "stream_id"); + + var traceId = runtimeEntityType.AddProperty( + "TraceId", + typeof(string), + propertyInfo: typeof(ObservationTimelineEventRow).GetProperty("TraceId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ObservationTimelineEventRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + traceId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + traceId.AddAnnotation("Relational:ColumnName", "trace_id"); + + var key = runtimeEntityType.AddKey( + new[] { tenant, eventId }); + runtimeEntityType.SetPrimaryKey(key); + key.AddAnnotation("Relational:Name", "observation_timeline_events_pkey"); + + var index = runtimeEntityType.AddIndex( + new[] { tenant }); + index.AddAnnotation("Relational:Name", "idx_obs_timeline_events_tenant"); + + var index0 = runtimeEntityType.AddIndex( + new[] { tenant, createdAt }); + index0.AddAnnotation("Relational:Name", "idx_obs_timeline_events_created_at"); + + var index1 = runtimeEntityType.AddIndex( + new[] { tenant, eventType }); + index1.AddAnnotation("Relational:Name", "idx_obs_timeline_events_type"); + + var index2 = runtimeEntityType.AddIndex( + new[] { tenant, providerId }); + index2.AddAnnotation("Relational:Name", "idx_obs_timeline_events_provider"); + + var index3 = runtimeEntityType.AddIndex( + new[] { tenant, traceId }); + index3.AddAnnotation("Relational:Name", "idx_obs_timeline_events_trace_id"); + + return runtimeEntityType; + } + + public static void CreateAnnotations(RuntimeEntityType runtimeEntityType) + { + runtimeEntityType.AddAnnotation("Relational:FunctionName", null); + runtimeEntityType.AddAnnotation("Relational:Schema", "vex"); + runtimeEntityType.AddAnnotation("Relational:SqlQuery", null); + runtimeEntityType.AddAnnotation("Relational:TableName", "observation_timeline_events"); + runtimeEntityType.AddAnnotation("Relational:ViewName", null); + runtimeEntityType.AddAnnotation("Relational:ViewSchema", null); + + Customize(runtimeEntityType); + } + + static partial void Customize(RuntimeEntityType runtimeEntityType); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ProviderRowEntityType.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ProviderRowEntityType.cs new file mode 100644 index 000000000..ee31e1726 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/ProviderRowEntityType.cs @@ -0,0 +1,147 @@ +// +using System; +using System.Reflection; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using StellaOps.Excititor.Persistence.EfCore.Models; + +#pragma warning disable 219, 612, 618 +#nullable disable + +namespace StellaOps.Excititor.Persistence.EfCore.CompiledModels +{ + [EntityFrameworkInternal] + public partial class ProviderRowEntityType + { + public static RuntimeEntityType Create(RuntimeModel model, RuntimeEntityType baseEntityType = null) + { + var runtimeEntityType = model.AddEntityType( + "StellaOps.Excititor.Persistence.EfCore.Models.ProviderRow", + typeof(ProviderRow), + baseEntityType, + propertyCount: 9, + unnamedIndexCount: 2, + keyCount: 1); + + var id = runtimeEntityType.AddProperty( + "Id", + typeof(string), + propertyInfo: typeof(ProviderRow).GetProperty("Id", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ProviderRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + afterSaveBehavior: PropertySaveBehavior.Throw); + id.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + id.AddAnnotation("Relational:ColumnName", "id"); + + var baseUris = runtimeEntityType.AddProperty( + "BaseUris", + typeof(string[]), + propertyInfo: typeof(ProviderRow).GetProperty("BaseUris", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ProviderRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + var baseUrisElementType = baseUris.SetElementType(typeof(string)); + baseUris.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + baseUris.AddAnnotation("Relational:ColumnName", "base_uris"); + + var createdAt = runtimeEntityType.AddProperty( + "CreatedAt", + typeof(DateTime), + propertyInfo: typeof(ProviderRow).GetProperty("CreatedAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ProviderRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + createdAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + createdAt.AddAnnotation("Relational:ColumnName", "created_at"); + createdAt.AddAnnotation("Relational:DefaultValueSql", "now()"); + + var discovery = runtimeEntityType.AddProperty( + "Discovery", + typeof(string), + propertyInfo: typeof(ProviderRow).GetProperty("Discovery", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ProviderRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd); + discovery.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + discovery.AddAnnotation("Relational:ColumnName", "discovery"); + discovery.AddAnnotation("Relational:ColumnType", "jsonb"); + discovery.AddAnnotation("Relational:DefaultValueSql", "'{}'::jsonb"); + + var displayName = runtimeEntityType.AddProperty( + "DisplayName", + typeof(string), + propertyInfo: typeof(ProviderRow).GetProperty("DisplayName", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ProviderRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + displayName.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + displayName.AddAnnotation("Relational:ColumnName", "display_name"); + + var enabled = runtimeEntityType.AddProperty( + "Enabled", + typeof(bool), + propertyInfo: typeof(ProviderRow).GetProperty("Enabled", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ProviderRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: true); + enabled.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + enabled.AddAnnotation("Relational:ColumnName", "enabled"); + enabled.AddAnnotation("Relational:DefaultValue", true); + + var kind = runtimeEntityType.AddProperty( + "Kind", + typeof(string), + propertyInfo: typeof(ProviderRow).GetProperty("Kind", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ProviderRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + kind.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + kind.AddAnnotation("Relational:ColumnName", "kind"); + + var trust = runtimeEntityType.AddProperty( + "Trust", + typeof(string), + propertyInfo: typeof(ProviderRow).GetProperty("Trust", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ProviderRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd); + trust.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + trust.AddAnnotation("Relational:ColumnName", "trust"); + trust.AddAnnotation("Relational:ColumnType", "jsonb"); + trust.AddAnnotation("Relational:DefaultValueSql", "'{}'::jsonb"); + + var updatedAt = runtimeEntityType.AddProperty( + "UpdatedAt", + typeof(DateTime), + propertyInfo: typeof(ProviderRow).GetProperty("UpdatedAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(ProviderRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + updatedAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + updatedAt.AddAnnotation("Relational:ColumnName", "updated_at"); + updatedAt.AddAnnotation("Relational:DefaultValueSql", "now()"); + + var key = runtimeEntityType.AddKey( + new[] { id }); + runtimeEntityType.SetPrimaryKey(key); + key.AddAnnotation("Relational:Name", "providers_pkey"); + + var index = runtimeEntityType.AddIndex( + new[] { enabled }); + index.AddAnnotation("Relational:Filter", "(enabled = true)"); + index.AddAnnotation("Relational:Name", "idx_providers_enabled"); + + var index0 = runtimeEntityType.AddIndex( + new[] { kind }); + index0.AddAnnotation("Relational:Name", "idx_providers_kind"); + + return runtimeEntityType; + } + + public static void CreateAnnotations(RuntimeEntityType runtimeEntityType) + { + runtimeEntityType.AddAnnotation("Relational:FunctionName", null); + runtimeEntityType.AddAnnotation("Relational:Schema", "vex"); + runtimeEntityType.AddAnnotation("Relational:SqlQuery", null); + runtimeEntityType.AddAnnotation("Relational:TableName", "providers"); + runtimeEntityType.AddAnnotation("Relational:ViewName", null); + runtimeEntityType.AddAnnotation("Relational:ViewSchema", null); + + Customize(runtimeEntityType); + } + + static partial void Customize(RuntimeEntityType runtimeEntityType); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/SourceTrustVectorEntityType.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/SourceTrustVectorEntityType.cs new file mode 100644 index 000000000..783ab6bbf --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/SourceTrustVectorEntityType.cs @@ -0,0 +1,133 @@ +// +using System; +using System.Reflection; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using StellaOps.Excititor.Persistence.EfCore.Models; + +#pragma warning disable 219, 612, 618 +#nullable disable + +namespace StellaOps.Excititor.Persistence.EfCore.CompiledModels +{ + [EntityFrameworkInternal] + public partial class SourceTrustVectorEntityType + { + public static RuntimeEntityType Create(RuntimeModel model, RuntimeEntityType baseEntityType = null) + { + var runtimeEntityType = model.AddEntityType( + "StellaOps.Excititor.Persistence.EfCore.Models.SourceTrustVector", + typeof(SourceTrustVector), + baseEntityType, + propertyCount: 8, + unnamedIndexCount: 2, + keyCount: 1); + + var id = runtimeEntityType.AddProperty( + "Id", + typeof(Guid), + propertyInfo: typeof(SourceTrustVector).GetProperty("Id", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(SourceTrustVector).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + afterSaveBehavior: PropertySaveBehavior.Throw, + sentinel: new Guid("00000000-0000-0000-0000-000000000000")); + id.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + id.AddAnnotation("Relational:ColumnName", "id"); + id.AddAnnotation("Relational:DefaultValueSql", "gen_random_uuid()"); + + var calibrationManifestId = runtimeEntityType.AddProperty( + "CalibrationManifestId", + typeof(string), + propertyInfo: typeof(SourceTrustVector).GetProperty("CalibrationManifestId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(SourceTrustVector).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + calibrationManifestId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + calibrationManifestId.AddAnnotation("Relational:ColumnName", "calibration_manifest_id"); + + var coverage = runtimeEntityType.AddProperty( + "Coverage", + typeof(double), + propertyInfo: typeof(SourceTrustVector).GetProperty("Coverage", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(SourceTrustVector).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: 0.0); + coverage.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + coverage.AddAnnotation("Relational:ColumnName", "coverage"); + + var provenance = runtimeEntityType.AddProperty( + "Provenance", + typeof(double), + propertyInfo: typeof(SourceTrustVector).GetProperty("Provenance", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(SourceTrustVector).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: 0.0); + provenance.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + provenance.AddAnnotation("Relational:ColumnName", "provenance"); + + var replayability = runtimeEntityType.AddProperty( + "Replayability", + typeof(double), + propertyInfo: typeof(SourceTrustVector).GetProperty("Replayability", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(SourceTrustVector).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: 0.0); + replayability.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + replayability.AddAnnotation("Relational:ColumnName", "replayability"); + + var sourceId = runtimeEntityType.AddProperty( + "SourceId", + typeof(string), + propertyInfo: typeof(SourceTrustVector).GetProperty("SourceId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(SourceTrustVector).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + sourceId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + sourceId.AddAnnotation("Relational:ColumnName", "source_id"); + + var tenant = runtimeEntityType.AddProperty( + "Tenant", + typeof(string), + propertyInfo: typeof(SourceTrustVector).GetProperty("Tenant", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(SourceTrustVector).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + tenant.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + tenant.AddAnnotation("Relational:ColumnName", "tenant"); + + var updatedAt = runtimeEntityType.AddProperty( + "UpdatedAt", + typeof(DateTime), + propertyInfo: typeof(SourceTrustVector).GetProperty("UpdatedAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(SourceTrustVector).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + updatedAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + updatedAt.AddAnnotation("Relational:ColumnName", "updated_at"); + updatedAt.AddAnnotation("Relational:DefaultValueSql", "now()"); + + var key = runtimeEntityType.AddKey( + new[] { id }); + runtimeEntityType.SetPrimaryKey(key); + key.AddAnnotation("Relational:Name", "source_trust_vectors_pkey"); + + var index = runtimeEntityType.AddIndex( + new[] { tenant }); + index.AddAnnotation("Relational:Name", "idx_source_vectors_tenant"); + + var index0 = runtimeEntityType.AddIndex( + new[] { tenant, sourceId }, + unique: true); + index0.AddAnnotation("Relational:Name", "source_trust_vectors_tenant_source_id_key"); + + return runtimeEntityType; + } + + public static void CreateAnnotations(RuntimeEntityType runtimeEntityType) + { + runtimeEntityType.AddAnnotation("Relational:FunctionName", null); + runtimeEntityType.AddAnnotation("Relational:Schema", "excititor"); + runtimeEntityType.AddAnnotation("Relational:SqlQuery", null); + runtimeEntityType.AddAnnotation("Relational:TableName", "source_trust_vectors"); + runtimeEntityType.AddAnnotation("Relational:ViewName", null); + runtimeEntityType.AddAnnotation("Relational:ViewSchema", null); + + Customize(runtimeEntityType); + } + + static partial void Customize(RuntimeEntityType runtimeEntityType); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/StatementRowEntityType.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/StatementRowEntityType.cs new file mode 100644 index 000000000..5b5a32e8e --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/StatementRowEntityType.cs @@ -0,0 +1,229 @@ +// +using System; +using System.Reflection; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using StellaOps.Excititor.Persistence.EfCore.Models; + +#pragma warning disable 219, 612, 618 +#nullable disable + +namespace StellaOps.Excititor.Persistence.EfCore.CompiledModels +{ + [EntityFrameworkInternal] + public partial class StatementRowEntityType + { + public static RuntimeEntityType Create(RuntimeModel model, RuntimeEntityType baseEntityType = null) + { + var runtimeEntityType = model.AddEntityType( + "StellaOps.Excititor.Persistence.EfCore.Models.StatementRow", + typeof(StatementRow), + baseEntityType, + propertyCount: 19, + keyCount: 1); + + var id = runtimeEntityType.AddProperty( + "Id", + typeof(Guid), + propertyInfo: typeof(StatementRow).GetProperty("Id", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(StatementRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + afterSaveBehavior: PropertySaveBehavior.Throw, + sentinel: new Guid("00000000-0000-0000-0000-000000000000")); + id.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + id.AddAnnotation("Relational:ColumnName", "id"); + id.AddAnnotation("Relational:DefaultValueSql", "gen_random_uuid()"); + + var actionStatement = runtimeEntityType.AddProperty( + "ActionStatement", + typeof(string), + propertyInfo: typeof(StatementRow).GetProperty("ActionStatement", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(StatementRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + actionStatement.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + actionStatement.AddAnnotation("Relational:ColumnName", "action_statement"); + + var actionStatementTimestamp = runtimeEntityType.AddProperty( + "ActionStatementTimestamp", + typeof(DateTime?), + propertyInfo: typeof(StatementRow).GetProperty("ActionStatementTimestamp", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(StatementRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + actionStatementTimestamp.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + actionStatementTimestamp.AddAnnotation("Relational:ColumnName", "action_statement_timestamp"); + + var createdBy = runtimeEntityType.AddProperty( + "CreatedBy", + typeof(string), + propertyInfo: typeof(StatementRow).GetProperty("CreatedBy", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(StatementRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + createdBy.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + createdBy.AddAnnotation("Relational:ColumnName", "created_by"); + + var evidence = runtimeEntityType.AddProperty( + "Evidence", + typeof(string), + propertyInfo: typeof(StatementRow).GetProperty("Evidence", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(StatementRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd); + evidence.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + evidence.AddAnnotation("Relational:ColumnName", "evidence"); + evidence.AddAnnotation("Relational:ColumnType", "jsonb"); + evidence.AddAnnotation("Relational:DefaultValueSql", "'{}'::jsonb"); + + var firstIssued = runtimeEntityType.AddProperty( + "FirstIssued", + typeof(DateTime), + propertyInfo: typeof(StatementRow).GetProperty("FirstIssued", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(StatementRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + firstIssued.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + firstIssued.AddAnnotation("Relational:ColumnName", "first_issued"); + firstIssued.AddAnnotation("Relational:DefaultValueSql", "now()"); + + var graphRevisionId = runtimeEntityType.AddProperty( + "GraphRevisionId", + typeof(Guid?), + propertyInfo: typeof(StatementRow).GetProperty("GraphRevisionId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(StatementRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + graphRevisionId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + graphRevisionId.AddAnnotation("Relational:ColumnName", "graph_revision_id"); + + var impactStatement = runtimeEntityType.AddProperty( + "ImpactStatement", + typeof(string), + propertyInfo: typeof(StatementRow).GetProperty("ImpactStatement", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(StatementRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + impactStatement.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + impactStatement.AddAnnotation("Relational:ColumnName", "impact_statement"); + + var justification = runtimeEntityType.AddProperty( + "Justification", + typeof(string), + propertyInfo: typeof(StatementRow).GetProperty("Justification", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(StatementRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + justification.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + justification.AddAnnotation("Relational:ColumnName", "justification"); + + var lastUpdated = runtimeEntityType.AddProperty( + "LastUpdated", + typeof(DateTime), + propertyInfo: typeof(StatementRow).GetProperty("LastUpdated", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(StatementRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + lastUpdated.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + lastUpdated.AddAnnotation("Relational:ColumnName", "last_updated"); + lastUpdated.AddAnnotation("Relational:DefaultValueSql", "now()"); + + var metadata = runtimeEntityType.AddProperty( + "Metadata", + typeof(string), + propertyInfo: typeof(StatementRow).GetProperty("Metadata", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(StatementRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd); + metadata.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + metadata.AddAnnotation("Relational:ColumnName", "metadata"); + metadata.AddAnnotation("Relational:ColumnType", "jsonb"); + metadata.AddAnnotation("Relational:DefaultValueSql", "'{}'::jsonb"); + + var productId = runtimeEntityType.AddProperty( + "ProductId", + typeof(string), + propertyInfo: typeof(StatementRow).GetProperty("ProductId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(StatementRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + productId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + productId.AddAnnotation("Relational:ColumnName", "product_id"); + + var projectId = runtimeEntityType.AddProperty( + "ProjectId", + typeof(Guid?), + propertyInfo: typeof(StatementRow).GetProperty("ProjectId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(StatementRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + projectId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + projectId.AddAnnotation("Relational:ColumnName", "project_id"); + + var provenance = runtimeEntityType.AddProperty( + "Provenance", + typeof(string), + propertyInfo: typeof(StatementRow).GetProperty("Provenance", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(StatementRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd); + provenance.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + provenance.AddAnnotation("Relational:ColumnName", "provenance"); + provenance.AddAnnotation("Relational:ColumnType", "jsonb"); + provenance.AddAnnotation("Relational:DefaultValueSql", "'{}'::jsonb"); + + var source = runtimeEntityType.AddProperty( + "Source", + typeof(string), + propertyInfo: typeof(StatementRow).GetProperty("Source", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(StatementRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + source.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + source.AddAnnotation("Relational:ColumnName", "source"); + + var sourceUrl = runtimeEntityType.AddProperty( + "SourceUrl", + typeof(string), + propertyInfo: typeof(StatementRow).GetProperty("SourceUrl", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(StatementRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + sourceUrl.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + sourceUrl.AddAnnotation("Relational:ColumnName", "source_url"); + + var status = runtimeEntityType.AddProperty( + "Status", + typeof(string), + propertyInfo: typeof(StatementRow).GetProperty("Status", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(StatementRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + status.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + status.AddAnnotation("Relational:ColumnName", "status"); + + var tenantId = runtimeEntityType.AddProperty( + "TenantId", + typeof(string), + propertyInfo: typeof(StatementRow).GetProperty("TenantId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(StatementRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + tenantId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + tenantId.AddAnnotation("Relational:ColumnName", "tenant_id"); + + var vulnerabilityId = runtimeEntityType.AddProperty( + "VulnerabilityId", + typeof(string), + propertyInfo: typeof(StatementRow).GetProperty("VulnerabilityId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(StatementRow).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + vulnerabilityId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + vulnerabilityId.AddAnnotation("Relational:ColumnName", "vulnerability_id"); + + var key = runtimeEntityType.AddKey( + new[] { id }); + runtimeEntityType.SetPrimaryKey(key); + key.AddAnnotation("Relational:Name", "statements_pkey"); + + return runtimeEntityType; + } + + public static void CreateAnnotations(RuntimeEntityType runtimeEntityType) + { + runtimeEntityType.AddAnnotation("Relational:FunctionName", null); + runtimeEntityType.AddAnnotation("Relational:Schema", "vex"); + runtimeEntityType.AddAnnotation("Relational:SqlQuery", null); + runtimeEntityType.AddAnnotation("Relational:TableName", "statements"); + runtimeEntityType.AddAnnotation("Relational:ViewName", null); + runtimeEntityType.AddAnnotation("Relational:ViewSchema", null); + + Customize(runtimeEntityType); + } + + static partial void Customize(RuntimeEntityType runtimeEntityType); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/VexRawBlobEntityType.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/VexRawBlobEntityType.cs new file mode 100644 index 000000000..a099cd1de --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/VexRawBlobEntityType.cs @@ -0,0 +1,73 @@ +// +using System; +using System.Reflection; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using StellaOps.Excititor.Persistence.EfCore.Models; + +#pragma warning disable 219, 612, 618 +#nullable disable + +namespace StellaOps.Excititor.Persistence.EfCore.CompiledModels +{ + [EntityFrameworkInternal] + public partial class VexRawBlobEntityType + { + public static RuntimeEntityType Create(RuntimeModel model, RuntimeEntityType baseEntityType = null) + { + var runtimeEntityType = model.AddEntityType( + "StellaOps.Excititor.Persistence.EfCore.Models.VexRawBlob", + typeof(VexRawBlob), + baseEntityType, + propertyCount: 3, + keyCount: 1); + + var digest = runtimeEntityType.AddProperty( + "Digest", + typeof(string), + propertyInfo: typeof(VexRawBlob).GetProperty("Digest", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawBlob).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + afterSaveBehavior: PropertySaveBehavior.Throw); + digest.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + digest.AddAnnotation("Relational:ColumnName", "digest"); + + var payload = runtimeEntityType.AddProperty( + "Payload", + typeof(byte[]), + propertyInfo: typeof(VexRawBlob).GetProperty("Payload", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawBlob).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + payload.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + payload.AddAnnotation("Relational:ColumnName", "payload"); + + var payloadHash = runtimeEntityType.AddProperty( + "PayloadHash", + typeof(string), + propertyInfo: typeof(VexRawBlob).GetProperty("PayloadHash", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawBlob).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + payloadHash.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + payloadHash.AddAnnotation("Relational:ColumnName", "payload_hash"); + + var key = runtimeEntityType.AddKey( + new[] { digest }); + runtimeEntityType.SetPrimaryKey(key); + key.AddAnnotation("Relational:Name", "vex_raw_blobs_pkey"); + + return runtimeEntityType; + } + + public static void CreateAnnotations(RuntimeEntityType runtimeEntityType) + { + runtimeEntityType.AddAnnotation("Relational:FunctionName", null); + runtimeEntityType.AddAnnotation("Relational:Schema", "vex"); + runtimeEntityType.AddAnnotation("Relational:SqlQuery", null); + runtimeEntityType.AddAnnotation("Relational:TableName", "vex_raw_blobs"); + runtimeEntityType.AddAnnotation("Relational:ViewName", null); + runtimeEntityType.AddAnnotation("Relational:ViewSchema", null); + + Customize(runtimeEntityType); + } + + static partial void Customize(RuntimeEntityType runtimeEntityType); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/VexRawDocumentEntityType.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/VexRawDocumentEntityType.cs new file mode 100644 index 000000000..96d765efd --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/EfCore/CompiledModels/VexRawDocumentEntityType.cs @@ -0,0 +1,234 @@ +// +using System; +using System.Reflection; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using StellaOps.Excititor.Persistence.EfCore.Models; + +#pragma warning disable 219, 612, 618 +#nullable disable + +namespace StellaOps.Excititor.Persistence.EfCore.CompiledModels +{ + [EntityFrameworkInternal] + public partial class VexRawDocumentEntityType + { + public static RuntimeEntityType Create(RuntimeModel model, RuntimeEntityType baseEntityType = null) + { + var runtimeEntityType = model.AddEntityType( + "StellaOps.Excititor.Persistence.EfCore.Models.VexRawDocument", + typeof(VexRawDocument), + baseEntityType, + propertyCount: 19, + keyCount: 1); + + var digest = runtimeEntityType.AddProperty( + "Digest", + typeof(string), + propertyInfo: typeof(VexRawDocument).GetProperty("Digest", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawDocument).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + afterSaveBehavior: PropertySaveBehavior.Throw); + digest.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + digest.AddAnnotation("Relational:ColumnName", "digest"); + + var contentJson = runtimeEntityType.AddProperty( + "ContentJson", + typeof(string), + propertyInfo: typeof(VexRawDocument).GetProperty("ContentJson", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawDocument).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + contentJson.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + contentJson.AddAnnotation("Relational:ColumnName", "content_json"); + contentJson.AddAnnotation("Relational:ColumnType", "jsonb"); + + var contentSizeBytes = runtimeEntityType.AddProperty( + "ContentSizeBytes", + typeof(int), + propertyInfo: typeof(VexRawDocument).GetProperty("ContentSizeBytes", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawDocument).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: 0); + contentSizeBytes.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + contentSizeBytes.AddAnnotation("Relational:ColumnName", "content_size_bytes"); + + var docAuthor = runtimeEntityType.AddProperty( + "DocAuthor", + typeof(string), + propertyInfo: typeof(VexRawDocument).GetProperty("DocAuthor", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawDocument).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true, + valueGenerated: ValueGenerated.OnAddOrUpdate, + beforeSaveBehavior: PropertySaveBehavior.Ignore, + afterSaveBehavior: PropertySaveBehavior.Ignore); + docAuthor.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + docAuthor.AddAnnotation("Relational:ColumnName", "doc_author"); + + var docFormatVersion = runtimeEntityType.AddProperty( + "DocFormatVersion", + typeof(string), + propertyInfo: typeof(VexRawDocument).GetProperty("DocFormatVersion", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawDocument).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true, + valueGenerated: ValueGenerated.OnAddOrUpdate, + beforeSaveBehavior: PropertySaveBehavior.Ignore, + afterSaveBehavior: PropertySaveBehavior.Ignore); + docFormatVersion.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + docFormatVersion.AddAnnotation("Relational:ColumnName", "doc_format_version"); + + var docTimestamp = runtimeEntityType.AddProperty( + "DocTimestamp", + typeof(string), + propertyInfo: typeof(VexRawDocument).GetProperty("DocTimestamp", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawDocument).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true, + valueGenerated: ValueGenerated.OnAddOrUpdate, + beforeSaveBehavior: PropertySaveBehavior.Ignore, + afterSaveBehavior: PropertySaveBehavior.Ignore); + docTimestamp.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + docTimestamp.AddAnnotation("Relational:ColumnName", "doc_timestamp"); + + var docToolName = runtimeEntityType.AddProperty( + "DocToolName", + typeof(string), + propertyInfo: typeof(VexRawDocument).GetProperty("DocToolName", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawDocument).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true, + valueGenerated: ValueGenerated.OnAddOrUpdate, + beforeSaveBehavior: PropertySaveBehavior.Ignore, + afterSaveBehavior: PropertySaveBehavior.Ignore); + docToolName.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + docToolName.AddAnnotation("Relational:ColumnName", "doc_tool_name"); + + var docToolVersion = runtimeEntityType.AddProperty( + "DocToolVersion", + typeof(string), + propertyInfo: typeof(VexRawDocument).GetProperty("DocToolVersion", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawDocument).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true, + valueGenerated: ValueGenerated.OnAddOrUpdate, + beforeSaveBehavior: PropertySaveBehavior.Ignore, + afterSaveBehavior: PropertySaveBehavior.Ignore); + docToolVersion.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + docToolVersion.AddAnnotation("Relational:ColumnName", "doc_tool_version"); + + var etag = runtimeEntityType.AddProperty( + "Etag", + typeof(string), + propertyInfo: typeof(VexRawDocument).GetProperty("Etag", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawDocument).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + etag.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + etag.AddAnnotation("Relational:ColumnName", "etag"); + + var format = runtimeEntityType.AddProperty( + "Format", + typeof(string), + propertyInfo: typeof(VexRawDocument).GetProperty("Format", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawDocument).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + format.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + format.AddAnnotation("Relational:ColumnName", "format"); + + var inlinePayload = runtimeEntityType.AddProperty( + "InlinePayload", + typeof(bool), + propertyInfo: typeof(VexRawDocument).GetProperty("InlinePayload", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawDocument).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: true); + inlinePayload.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + inlinePayload.AddAnnotation("Relational:ColumnName", "inline_payload"); + inlinePayload.AddAnnotation("Relational:DefaultValue", true); + + var metadataJson = runtimeEntityType.AddProperty( + "MetadataJson", + typeof(string), + propertyInfo: typeof(VexRawDocument).GetProperty("MetadataJson", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawDocument).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + metadataJson.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + metadataJson.AddAnnotation("Relational:ColumnName", "metadata_json"); + metadataJson.AddAnnotation("Relational:ColumnType", "jsonb"); + + var provenanceJson = runtimeEntityType.AddProperty( + "ProvenanceJson", + typeof(string), + propertyInfo: typeof(VexRawDocument).GetProperty("ProvenanceJson", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawDocument).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + provenanceJson.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + provenanceJson.AddAnnotation("Relational:ColumnName", "provenance_json"); + provenanceJson.AddAnnotation("Relational:ColumnType", "jsonb"); + + var providerId = runtimeEntityType.AddProperty( + "ProviderId", + typeof(string), + propertyInfo: typeof(VexRawDocument).GetProperty("ProviderId", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawDocument).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + providerId.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + providerId.AddAnnotation("Relational:ColumnName", "provider_id"); + + var recordedAt = runtimeEntityType.AddProperty( + "RecordedAt", + typeof(DateTime), + propertyInfo: typeof(VexRawDocument).GetProperty("RecordedAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawDocument).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + valueGenerated: ValueGenerated.OnAdd, + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + recordedAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + recordedAt.AddAnnotation("Relational:ColumnName", "recorded_at"); + recordedAt.AddAnnotation("Relational:DefaultValueSql", "now()"); + + var retrievedAt = runtimeEntityType.AddProperty( + "RetrievedAt", + typeof(DateTime), + propertyInfo: typeof(VexRawDocument).GetProperty("RetrievedAt", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawDocument).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + sentinel: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + retrievedAt.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + retrievedAt.AddAnnotation("Relational:ColumnName", "retrieved_at"); + + var sourceUri = runtimeEntityType.AddProperty( + "SourceUri", + typeof(string), + propertyInfo: typeof(VexRawDocument).GetProperty("SourceUri", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawDocument).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + sourceUri.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + sourceUri.AddAnnotation("Relational:ColumnName", "source_uri"); + + var supersedesDigest = runtimeEntityType.AddProperty( + "SupersedesDigest", + typeof(string), + propertyInfo: typeof(VexRawDocument).GetProperty("SupersedesDigest", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawDocument).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly), + nullable: true); + supersedesDigest.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + supersedesDigest.AddAnnotation("Relational:ColumnName", "supersedes_digest"); + + var tenant = runtimeEntityType.AddProperty( + "Tenant", + typeof(string), + propertyInfo: typeof(VexRawDocument).GetProperty("Tenant", BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly), + fieldInfo: typeof(VexRawDocument).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)); + tenant.AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.None); + tenant.AddAnnotation("Relational:ColumnName", "tenant"); + + var key = runtimeEntityType.AddKey( + new[] { digest }); + runtimeEntityType.SetPrimaryKey(key); + key.AddAnnotation("Relational:Name", "vex_raw_documents_pkey"); + + return runtimeEntityType; + } + + public static void CreateAnnotations(RuntimeEntityType runtimeEntityType) + { + runtimeEntityType.AddAnnotation("Relational:FunctionName", null); + runtimeEntityType.AddAnnotation("Relational:Schema", "vex"); + runtimeEntityType.AddAnnotation("Relational:SqlQuery", null); + runtimeEntityType.AddAnnotation("Relational:TableName", "vex_raw_documents"); + runtimeEntityType.AddAnnotation("Relational:ViewName", null); + runtimeEntityType.AddAnnotation("Relational:ViewSchema", null); + + Customize(runtimeEntityType); + } + + static partial void Customize(RuntimeEntityType runtimeEntityType); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Extensions/ExcititorPersistenceExtensions.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Extensions/ExcititorPersistenceExtensions.cs index 7c240ab5e..52ef208f7 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Extensions/ExcititorPersistenceExtensions.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Extensions/ExcititorPersistenceExtensions.cs @@ -55,6 +55,7 @@ public static class ExcititorPersistenceExtensions services.AddScoped(); // Register VEX auxiliary stores (SPRINT-3412: PostgreSQL durability) + services.AddSingleton(); services.AddScoped(); services.AddScoped(); services.AddScoped(); @@ -97,6 +98,7 @@ public static class ExcititorPersistenceExtensions services.AddScoped(); // Register VEX auxiliary stores (SPRINT-3412: PostgreSQL durability) + services.AddSingleton(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Migrations/006_vex_runtime_projection_tables.sql b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Migrations/006_vex_runtime_projection_tables.sql new file mode 100644 index 000000000..0b21b40db --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Migrations/006_vex_runtime_projection_tables.sql @@ -0,0 +1,186 @@ +-- Migration: 006_vex_runtime_projection_tables +-- Category: startup +-- Description: Create the current Excititor runtime projection tables required by the live repositories. + +CREATE TABLE IF NOT EXISTS vex.checkpoint_mutations ( + sequence_number BIGSERIAL PRIMARY KEY, + tenant_id TEXT NOT NULL, + connector_id TEXT NOT NULL, + mutation_type TEXT NOT NULL, + run_id UUID NOT NULL, + timestamp TIMESTAMPTZ NOT NULL, + cursor TEXT, + artifact_hash TEXT, + artifact_kind TEXT, + documents_processed INTEGER, + claims_generated INTEGER, + error_code TEXT, + error_message TEXT, + retry_after_seconds INTEGER, + idempotency_key TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_checkpoint_mutations_tenant_connector + ON vex.checkpoint_mutations (tenant_id, connector_id, sequence_number); + +CREATE TABLE IF NOT EXISTS vex.checkpoint_states ( + tenant_id TEXT NOT NULL, + connector_id TEXT NOT NULL, + cursor TEXT, + last_updated TIMESTAMPTZ, + last_run_id UUID, + last_mutation_type TEXT, + last_artifact_hash TEXT, + last_artifact_kind TEXT, + total_documents_processed INTEGER NOT NULL DEFAULT 0, + total_claims_generated INTEGER NOT NULL DEFAULT 0, + success_count INTEGER NOT NULL DEFAULT 0, + failure_count INTEGER NOT NULL DEFAULT 0, + last_error_code TEXT, + next_eligible_run TIMESTAMPTZ, + latest_sequence_number BIGINT NOT NULL DEFAULT 0, + CONSTRAINT checkpoint_states_pkey PRIMARY KEY (tenant_id, connector_id) +); + +CREATE TABLE IF NOT EXISTS vex.connector_states ( + connector_id TEXT NOT NULL, + last_updated TIMESTAMPTZ NOT NULL, + document_digests TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[], + resume_tokens JSONB NOT NULL DEFAULT '{}'::jsonb, + last_success_at TIMESTAMPTZ, + failure_count INTEGER NOT NULL DEFAULT 0, + next_eligible_run TIMESTAMPTZ, + last_failure_reason TEXT, + last_checkpoint TIMESTAMPTZ, + last_heartbeat_at TIMESTAMPTZ, + last_heartbeat_status TEXT, + last_artifact_hash TEXT, + last_artifact_kind TEXT, + CONSTRAINT connector_states_pkey PRIMARY KEY (connector_id) +); + +CREATE TABLE IF NOT EXISTS vex.deltas ( + id UUID NOT NULL DEFAULT gen_random_uuid(), + from_artifact_digest TEXT NOT NULL, + to_artifact_digest TEXT NOT NULL, + cve TEXT NOT NULL, + from_status TEXT NOT NULL, + to_status TEXT NOT NULL, + rationale JSONB, + replay_hash TEXT, + attestation_digest TEXT, + tenant_id TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT deltas_pkey PRIMARY KEY (id) +); + +CREATE UNIQUE INDEX IF NOT EXISTS uq_vex_delta + ON vex.deltas (from_artifact_digest, to_artifact_digest, cve, tenant_id); + +CREATE TABLE IF NOT EXISTS vex.providers ( + id TEXT NOT NULL, + display_name TEXT NOT NULL, + kind TEXT NOT NULL, + base_uris TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[], + discovery JSONB NOT NULL DEFAULT '{}'::jsonb, + trust JSONB NOT NULL DEFAULT '{}'::jsonb, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT providers_pkey PRIMARY KEY (id) +); + +CREATE INDEX IF NOT EXISTS idx_providers_kind + ON vex.providers (kind); + +CREATE INDEX IF NOT EXISTS idx_providers_enabled + ON vex.providers (enabled) + WHERE enabled = TRUE; + +CREATE TABLE IF NOT EXISTS vex.observation_timeline_events ( + event_id TEXT NOT NULL, + tenant TEXT NOT NULL, + provider_id TEXT NOT NULL, + stream_id TEXT NOT NULL, + event_type TEXT NOT NULL, + trace_id TEXT NOT NULL, + justification_summary TEXT NOT NULL DEFAULT '', + evidence_hash TEXT, + payload_hash TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + attributes JSONB NOT NULL DEFAULT '{}'::jsonb, + CONSTRAINT observation_timeline_events_pkey PRIMARY KEY (tenant, event_id) +); + +CREATE INDEX IF NOT EXISTS idx_obs_timeline_events_tenant + ON vex.observation_timeline_events (tenant); + +CREATE INDEX IF NOT EXISTS idx_obs_timeline_events_trace_id + ON vex.observation_timeline_events (tenant, trace_id); + +CREATE INDEX IF NOT EXISTS idx_obs_timeline_events_provider + ON vex.observation_timeline_events (tenant, provider_id); + +CREATE INDEX IF NOT EXISTS idx_obs_timeline_events_type + ON vex.observation_timeline_events (tenant, event_type); + +CREATE INDEX IF NOT EXISTS idx_obs_timeline_events_created_at + ON vex.observation_timeline_events (tenant, created_at DESC); + +CREATE TABLE IF NOT EXISTS vex.observations ( + observation_id TEXT NOT NULL, + tenant TEXT NOT NULL, + provider_id TEXT NOT NULL, + stream_id TEXT NOT NULL, + upstream JSONB NOT NULL, + statements JSONB NOT NULL DEFAULT '[]'::jsonb, + content JSONB NOT NULL, + linkset JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + supersedes TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[], + attributes JSONB NOT NULL DEFAULT '{}'::jsonb, + rekor_uuid TEXT, + rekor_log_index BIGINT, + rekor_integrated_time TIMESTAMPTZ, + rekor_log_url TEXT, + rekor_tree_root TEXT, + rekor_tree_size BIGINT, + rekor_inclusion_proof JSONB, + rekor_entry_body_hash TEXT, + rekor_entry_kind TEXT, + rekor_linked_at TIMESTAMPTZ, + CONSTRAINT observations_pkey PRIMARY KEY (tenant, observation_id) +); + +CREATE INDEX IF NOT EXISTS idx_observations_tenant + ON vex.observations (tenant); + +CREATE INDEX IF NOT EXISTS idx_observations_provider + ON vex.observations (tenant, provider_id); + +CREATE INDEX IF NOT EXISTS idx_observations_created_at + ON vex.observations (tenant, created_at DESC); + +CREATE TABLE IF NOT EXISTS vex.statements ( + id UUID NOT NULL DEFAULT gen_random_uuid(), + tenant_id TEXT NOT NULL, + project_id UUID, + graph_revision_id UUID, + vulnerability_id TEXT NOT NULL, + product_id TEXT, + status TEXT NOT NULL, + justification TEXT, + impact_statement TEXT, + action_statement TEXT, + action_statement_timestamp TIMESTAMPTZ, + first_issued TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_updated TIMESTAMPTZ NOT NULL DEFAULT NOW(), + source TEXT, + source_url TEXT, + evidence JSONB NOT NULL DEFAULT '{}'::jsonb, + provenance JSONB NOT NULL DEFAULT '{}'::jsonb, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_by TEXT, + CONSTRAINT statements_pkey PRIMARY KEY (id) +); diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Postgres/Repositories/PostgresAppendOnlyCheckpointStore.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Postgres/Repositories/PostgresAppendOnlyCheckpointStore.cs index ca7acc720..c9c53dc8d 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Postgres/Repositories/PostgresAppendOnlyCheckpointStore.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Postgres/Repositories/PostgresAppendOnlyCheckpointStore.cs @@ -228,7 +228,8 @@ public sealed class PostgresAppendOnlyCheckpointStore : RepositoryBase, AddParameter(command, "tenant", _tenantId); AddParameter(command, "vulnerability_id", vulnerabilityId.Trim()); AddParameter(command, "product_key", productKey.Trim()); - AddParameter(command, "since", since); + AddNullableTimestampParameter(command, "since", since); }, cancellationToken).ConfigureAwait(false); } @@ -278,8 +278,7 @@ public sealed class PostgresVexClaimStore : RepositoryBase, { var vulnerabilityId = reader.GetString(0); var providerId = reader.GetString(1); - var product = JsonSerializer.Deserialize(reader.GetString(2), SerializerOptions) - ?? throw new InvalidOperationException("Stored claim product payload is invalid."); + var product = DeserializeProduct(reader.GetString(2)); var status = ParseStatus(reader.GetString(3)); var justification = ParseJustification(GetNullableString(reader, 4)); var detail = GetNullableString(reader, 5); @@ -306,6 +305,44 @@ public sealed class PostgresVexClaimStore : RepositoryBase, additionalMetadata); } + private static void AddNullableTimestampParameter( + NpgsqlCommand command, + string name, + DateTimeOffset? value) + { + command.Parameters.Add(new NpgsqlParameter(name, NpgsqlDbType.TimestampTz) + { + TypedValue = value + }); + } + + private static VexProduct DeserializeProduct(string json) + { + using var document = JsonDocument.Parse(json); + var root = document.RootElement; + + var key = root.GetProperty("key").GetString() + ?? throw new InvalidOperationException("Stored claim product payload is missing key."); + var name = GetOptionalString(root, "name"); + var version = GetOptionalString(root, "version"); + var purl = GetOptionalString(root, "purl"); + var cpe = GetOptionalString(root, "cpe"); + + IEnumerable? componentIdentifiers = null; + if (root.TryGetProperty("componentIdentifiers", out var identifiersElement) && + identifiersElement.ValueKind == JsonValueKind.Array) + { + componentIdentifiers = identifiersElement + .EnumerateArray() + .Where(static item => item.ValueKind == JsonValueKind.String) + .Select(static item => item.GetString()!) + .Where(static item => !string.IsNullOrWhiteSpace(item)) + .ToArray(); + } + + return new VexProduct(key, name, version, purl, cpe, componentIdentifiers); + } + private static T? DeserializeOptional(NpgsqlDataReader reader, int ordinal) where T : class => reader.IsDBNull(ordinal) ? null @@ -324,6 +361,16 @@ public sealed class PostgresVexClaimStore : RepositoryBase, : metadata.ToImmutableDictionary(StringComparer.Ordinal); } + private static string? GetOptionalString(JsonElement element, string propertyName) + { + if (!element.TryGetProperty(propertyName, out var property) || property.ValueKind != JsonValueKind.String) + { + return null; + } + + return property.GetString(); + } + private static string ComputeSha256(string payload) { var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(payload)); diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Postgres/Repositories/PostgresVexEvidenceLinkStore.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Postgres/Repositories/PostgresVexEvidenceLinkStore.cs new file mode 100644 index 000000000..3ac4e18f2 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Postgres/Repositories/PostgresVexEvidenceLinkStore.cs @@ -0,0 +1,229 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Npgsql; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Core.Evidence; +using StellaOps.Excititor.Persistence.EfCore.Models; +using StellaOps.Infrastructure.Postgres.Repositories; +using System.Collections.Immutable; +using System.Text.Json; + +namespace StellaOps.Excititor.Persistence.Postgres.Repositories; + +/// +/// PostgreSQL-backed store for VEX evidence links using the durable vex.evidence_links table. +/// +public sealed class PostgresVexEvidenceLinkStore : RepositoryBase, IVexEvidenceLinkStore +{ + public PostgresVexEvidenceLinkStore( + ExcititorDataSource dataSource, + ILogger logger) + : base(dataSource, logger) + { + } + + public async Task SaveAsync(VexEvidenceLink link, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(link); + ct.ThrowIfCancellationRequested(); + + await using var connection = await DataSource.OpenConnectionAsync("public", "writer", ct).ConfigureAwait(false); + await using var dbContext = ExcititorDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName()); + + dbContext.EvidenceLinks.Add(ToEntity(link)); + + try + { + await dbContext.SaveChangesAsync(ct).ConfigureAwait(false); + } + catch (DbUpdateException ex) when (IsUniqueViolation(ex)) + { + // The link ID is deterministic. Duplicate inserts are idempotent no-ops. + dbContext.ChangeTracker.Clear(); + } + } + + public async Task GetAsync(string linkId, CancellationToken ct = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(linkId); + ct.ThrowIfCancellationRequested(); + + await using var connection = await DataSource.OpenConnectionAsync("public", "reader", ct).ConfigureAwait(false); + await using var dbContext = ExcititorDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName()); + + var row = await dbContext.EvidenceLinks + .AsNoTracking() + .FirstOrDefaultAsync(link => link.LinkId == linkId.Trim(), ct) + .ConfigureAwait(false); + + return row is null ? null : Map(row); + } + + public async Task> GetByVexEntryAsync(string vexEntryId, CancellationToken ct = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(vexEntryId); + ct.ThrowIfCancellationRequested(); + + await using var connection = await DataSource.OpenConnectionAsync("public", "reader", ct).ConfigureAwait(false); + await using var dbContext = ExcititorDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName()); + + var rows = await dbContext.EvidenceLinks + .AsNoTracking() + .Where(link => link.VexEntryId == vexEntryId.Trim()) + .OrderByDescending(link => link.Confidence) + .ThenBy(link => link.LinkedAt) + .ThenBy(link => link.LinkId) + .ToListAsync(ct) + .ConfigureAwait(false); + + return rows.Select(Map).ToImmutableArray(); + } + + public async Task DeleteAsync(string linkId, CancellationToken ct = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(linkId); + ct.ThrowIfCancellationRequested(); + + await using var connection = await DataSource.OpenConnectionAsync("public", "writer", ct).ConfigureAwait(false); + await using var dbContext = ExcititorDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName()); + + await dbContext.EvidenceLinks + .Where(link => link.LinkId == linkId.Trim()) + .ExecuteDeleteAsync(ct) + .ConfigureAwait(false); + } + + private static EvidenceLink ToEntity(VexEvidenceLink link) + { + return new EvidenceLink + { + LinkId = link.LinkId, + VexEntryId = link.VexEntryId, + EvidenceType = link.EvidenceType.ToString(), + EvidenceUri = link.EvidenceUri, + EnvelopeDigest = link.EnvelopeDigest, + PredicateType = link.PredicateType ?? string.Empty, + Confidence = link.Confidence, + Justification = link.Justification.ToString(), + EvidenceCreatedAt = link.EvidenceCreatedAt.UtcDateTime, + LinkedAt = link.LinkedAt.UtcDateTime, + SignerIdentity = link.SignerIdentity, + RekorLogIndex = link.RekorLogIndex, + SignatureValidated = link.SignatureValidated, + Metadata = SerializeMetadata(link.Metadata) + }; + } + + private static VexEvidenceLink Map(EvidenceLink row) + { + return new VexEvidenceLink + { + LinkId = row.LinkId, + VexEntryId = row.VexEntryId, + EvidenceType = ParseEvidenceType(row.EvidenceType), + EvidenceUri = row.EvidenceUri, + EnvelopeDigest = row.EnvelopeDigest, + PredicateType = row.PredicateType, + Confidence = row.Confidence, + Justification = ParseJustification(row.Justification), + EvidenceCreatedAt = new DateTimeOffset(DateTime.SpecifyKind(row.EvidenceCreatedAt, DateTimeKind.Utc)), + LinkedAt = new DateTimeOffset(DateTime.SpecifyKind(row.LinkedAt, DateTimeKind.Utc)), + SignerIdentity = row.SignerIdentity, + RekorLogIndex = row.RekorLogIndex, + SignatureValidated = row.SignatureValidated, + Metadata = DeserializeMetadata(row.Metadata) + }; + } + + private static string SerializeMetadata(ImmutableDictionary metadata) + { + if (metadata.IsEmpty) + { + return "{}"; + } + + var ordered = metadata + .Where(static pair => !string.IsNullOrWhiteSpace(pair.Key)) + .OrderBy(static pair => pair.Key, StringComparer.Ordinal) + .ToDictionary( + static pair => pair.Key.Trim(), + static pair => pair.Value?.Trim() ?? string.Empty, + StringComparer.Ordinal); + + return JsonSerializer.Serialize(ordered); + } + + private static ImmutableDictionary DeserializeMetadata(string? json) + { + if (string.IsNullOrWhiteSpace(json)) + { + return ImmutableDictionary.Empty; + } + + try + { + using var document = JsonDocument.Parse(json); + if (document.RootElement.ValueKind != JsonValueKind.Object) + { + return ImmutableDictionary.Empty; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var property in document.RootElement.EnumerateObject().OrderBy(static property => property.Name, StringComparer.Ordinal)) + { + if (string.IsNullOrWhiteSpace(property.Name)) + { + continue; + } + + builder[property.Name] = property.Value.ValueKind == JsonValueKind.String + ? property.Value.GetString() ?? string.Empty + : property.Value.GetRawText(); + } + + return builder.ToImmutable(); + } + catch + { + return ImmutableDictionary.Empty; + } + } + + private static EvidenceType ParseEvidenceType(string value) + { + if (Enum.TryParse(value, ignoreCase: true, out var parsed)) + { + return parsed; + } + + return EvidenceType.Other; + } + + private static VexJustification ParseJustification(string value) + { + if (Enum.TryParse(value, ignoreCase: true, out var parsed)) + { + return parsed; + } + + throw new InvalidOperationException($"Unsupported evidence-link justification value '{value}'."); + } + + private static string GetSchemaName() => ExcititorDataSource.DefaultSchemaName; + + private static bool IsUniqueViolation(DbUpdateException exception) + { + Exception? current = exception; + while (current is not null) + { + if (current is PostgresException { SqlState: PostgresErrorCodes.UniqueViolation }) + { + return true; + } + + current = current.InnerException; + } + + return false; + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Postgres/Repositories/PostgresVexObservationStore.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Postgres/Repositories/PostgresVexObservationStore.cs index 205ba12a5..f790da1d5 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Postgres/Repositories/PostgresVexObservationStore.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Postgres/Repositories/PostgresVexObservationStore.cs @@ -69,7 +69,8 @@ public sealed class PostgresVexObservationStore : RepositoryBase() : observation.Supersedes.ToArray(), - SerializeAttributes(observation.Attributes), - cancellationToken).ConfigureAwait(false); + SerializeAttributes(observation.Attributes) + }; + + await dbContext.Database.ExecuteSqlRawAsync(upsertSql, parameters, cancellationToken).ConfigureAwait(false); return true; } diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/TASKS.md b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/TASKS.md index a26a71146..72a8b7385 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/TASKS.md +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/TASKS.md @@ -12,3 +12,5 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | QA-DEVOPS-VERIFY-002-F | DONE | 2026-02-11: Fixed Rekor-linkage schema mismatch in `PostgresVexObservationStore` by aligning to `vex.observations` and ensuring Rekor linkage columns/indexes. | | NOMOCK-012 | DONE | 2026-04-14: Added PostgreSQL `IVexClaimStore`, wired startup migrations for `vex`, removed live demo-seed SQL, and restricted embedded Excititor migrations to active top-level files only. | | REALPLAN-007-A | DONE | 2026-04-15: Added startup migration `005_vex_attestations.sql` so `PostgresVexAttestationStore` owns a durable attestation table in the active runtime schema. | +| REALPLAN-007-D | DONE | 2026-04-20: Added `PostgresVexEvidenceLinkStore` over `vex.evidence_links`, registered it from `AddExcititorPersistence(...)`, and regenerated the compiled model so live hosts use durable evidence-link persistence. | +| CONN-ALIGN-003 | DOING | 2026-04-21: Backfilling the missing runtime projection tables (`vex.providers`, `vex.observations`, `vex.observation_timeline_events`, checkpoint state, deltas, and statements) into the active startup migration bundle so live Excititor connectors can persist seeded providers and fetched VEX results. | diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Connector.CertFr.Tests/CertFr/CertFrConnectorTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.Connector.CertFr.Tests/CertFr/CertFrConnectorTests.cs index fb8bb07e9..4dbefbf96 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.Connector.CertFr.Tests/CertFr/CertFrConnectorTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.Connector.CertFr.Tests/CertFr/CertFrConnectorTests.cs @@ -23,7 +23,7 @@ namespace StellaOps.Concelier.Connector.CertFr.Tests; [Collection(ConcelierFixtureCollection.Name)] public sealed class CertFrConnectorTests { - private static readonly Uri FeedUri = new("https://www.cert.ssi.gouv.fr/feed/alertes/"); + private static readonly Uri FeedUri = new("https://www.cert.ssi.gouv.fr/alerte/feed/"); private static readonly Uri FirstDetailUri = new("https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/"); private static readonly Uri SecondDetailUri = new("https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/"); diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Kev.Tests/Kev/KevConnectorTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Kev.Tests/Kev/KevConnectorTests.cs index 8271e379c..f178f8e52 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Kev.Tests/Kev/KevConnectorTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Kev.Tests/Kev/KevConnectorTests.cs @@ -7,6 +7,8 @@ using Microsoft.Extensions.Http; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; +using Npgsql; +using StellaOps.Concelier.Core.Canonical; using StellaOps.Concelier.Documents; using StellaOps.Concelier.Models; using StellaOps.Concelier.Connector.Common; @@ -17,6 +19,7 @@ using StellaOps.Concelier.Connector.Kev.Configuration; using StellaOps.Concelier.Storage; using StellaOps.Concelier.Storage.Advisories; using StellaOps.Concelier.Persistence.Postgres; +using StellaOps.Concelier.Persistence.Postgres.Repositories; using StellaOps.Concelier.Testing; using Xunit; @@ -68,7 +71,7 @@ public sealed class KevConnectorTests : IAsyncLifetime await connector.FetchAsync(provider, CancellationToken.None); _handler.AssertNoPendingResponses(); - var stateRepository = provider.GetRequiredService(); + var stateRepository = provider.GetRequiredService(); var state = await stateRepository.TryGetAsync(KevConnectorPlugin.SourceName, CancellationToken.None); Assert.NotNull(state); var stateValue = state!; @@ -78,7 +81,111 @@ public sealed class KevConnectorTests : IAsyncLifetime Assert.True(IsEmptyArray(stateValue.Cursor, "pendingMappings")); } - private async Task BuildServiceProviderAsync() + [Fact] + public async Task FetchParseMap_BindsSourceAndCanonicalProjection() + { + await using var provider = await BuildServiceProviderAsync(enableCanonicalIngest: true); + SeedCatalogResponse(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var sourceRepository = provider.GetRequiredService(); + var source = await sourceRepository.GetByKeyAsync(KevConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(source); + + var advisoryStore = provider.GetRequiredService(); + var advisoryRepository = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + var advisoryKey = advisories + .OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal) + .First() + .AdvisoryKey; + + var persisted = await advisoryRepository.GetByKeyAsync(advisoryKey, CancellationToken.None); + Assert.NotNull(persisted); + Assert.Equal(source!.Id, persisted!.SourceId); + + var canonicalStore = provider.GetRequiredService(); + var canonicalCount = await canonicalStore.CountAsync(CancellationToken.None); + Assert.True(canonicalCount > 0); + + var sourceReadRepository = provider.GetRequiredService(); + var sourceRecord = await sourceReadRepository.GetBySourceIdAsync(source.Id, CancellationToken.None); + Assert.NotNull(sourceRecord); + Assert.True(sourceRecord!.SourceDocumentCount > 0); + Assert.True(sourceRecord.CanonicalAdvisoryCount > 0); + Assert.True(sourceRecord.CveCount > 0); + } + + [Fact] + public async Task ForceMap_ReprocessesLatestCachedCatalogWhenNoPendingMappings() + { + await using var provider = await BuildServiceProviderAsync(enableCanonicalIngest: true); + SeedCatalogResponse(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var sourceRepository = provider.GetRequiredService(); + var advisoryRepository = provider.GetRequiredService(); + var sourceReadRepository = provider.GetRequiredService(); + var source = await sourceRepository.GetByKeyAsync(KevConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(source); + + var advisory = (await advisoryRepository.GetRecentAsync(1, CancellationToken.None)).Single(); + Assert.NotNull(advisory.SourceId); + + await using (var connection = new NpgsqlConnection(_fixture.ConnectionString)) + { + await connection.OpenAsync(CancellationToken.None); + await using var command = new NpgsqlCommand( + """ + UPDATE vuln.advisories + SET source_id = NULL + WHERE advisory_key LIKE 'kev/%'; + + DELETE FROM vuln.advisory_source_edge + WHERE source_id = @source_id; + """, + connection); + command.Parameters.AddWithValue("source_id", source!.Id); + await command.ExecuteNonQueryAsync(CancellationToken.None); + } + + var sourceRecordBefore = await sourceReadRepository.GetBySourceIdAsync(source.Id, CancellationToken.None); + Assert.NotNull(sourceRecordBefore); + Assert.Equal(0, sourceRecordBefore!.SourceDocumentCount); + Assert.Equal(0, sourceRecordBefore.CanonicalAdvisoryCount); + Assert.Equal(0, sourceRecordBefore.CveCount); + + var forceMap = typeof(KevConnector).GetMethod( + nameof(KevConnector.MapAsync), + new[] { typeof(IServiceProvider), typeof(bool), typeof(CancellationToken) }); + Assert.NotNull(forceMap); + + var replayTask = (Task?)forceMap!.Invoke(connector, new object?[] { provider, true, CancellationToken.None }); + Assert.NotNull(replayTask); + await replayTask!; + + var restored = await advisoryRepository.GetByKeyAsync(advisory.AdvisoryKey, CancellationToken.None); + Assert.NotNull(restored); + Assert.Equal(source.Id, restored!.SourceId); + + var sourceRecordAfter = await sourceReadRepository.GetBySourceIdAsync(source.Id, CancellationToken.None); + Assert.NotNull(sourceRecordAfter); + Assert.True(sourceRecordAfter!.SourceDocumentCount > 0); + Assert.True(sourceRecordAfter.CanonicalAdvisoryCount > 0); + Assert.True(sourceRecordAfter.CveCount > 0); + } + + private async Task BuildServiceProviderAsync(bool enableCanonicalIngest = false) { await _fixture.TruncateAllTablesAsync(CancellationToken.None); _handler.Clear(); @@ -101,6 +208,15 @@ public sealed class KevConnectorTests : IAsyncLifetime options.RequestTimeout = TimeSpan.FromSeconds(10); }); + if (enableCanonicalIngest) + { + services.AddScoped(sp => new CanonicalAdvisoryService( + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService>(), + sp.GetRequiredService())); + } + services.Configure(KevOptions.HttpClientName, builderOptions => { builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler); @@ -229,6 +345,3 @@ public sealed class KevConnectorTests : IAsyncLifetime await _fixture.TruncateAllTablesAsync(CancellationToken.None); } } - - - diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Kev.Tests/TASKS.md b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Kev.Tests/TASKS.md index e8f1bd02e..3dd06c1ab 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Kev.Tests/TASKS.md +++ b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Kev.Tests/TASKS.md @@ -1,7 +1,7 @@ # KEV Connector Tests Task Board This board mirrors active sprint tasks for this module. -Source of truth: `docs/implplan/SPRINT_0127_001_QA_test_stabilization.md` (current); `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229_049_BE_csproj_audit_maint_tests.md` (historical). +Source of truth: `docs/implplan/SPRINT_20260421_003_Concelier_advisory_connector_runtime_alignment.md` (current); `docs/implplan/SPRINT_0127_001_QA_test_stabilization.md` and `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229_049_BE_csproj_audit_maint_tests.md` (historical). | Task ID | Status | Notes | | --- | --- | --- | @@ -9,3 +9,4 @@ Source of truth: `docs/implplan/SPRINT_0127_001_QA_test_stabilization.md` (curre | AUDIT-0184-M | DONE | Revalidated 2026-01-06; findings recorded in audit report. | | AUDIT-0184-T | DONE | Revalidated 2026-01-06; findings recorded in audit report. | | AUDIT-0184-A | DONE | Waived (test project; revalidated 2026-01-06). | +| CONN-ALIGN-004 | DONE | 2026-04-22: Added forced cached-catalog replay regression coverage for KEV source binding and canonical/source-edge recovery; targeted `KevConnectorTests` helper run passed 3/3. | diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Osv.Tests/Osv/OsvConnectorResumeWindowTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Osv.Tests/Osv/OsvConnectorResumeWindowTests.cs new file mode 100644 index 000000000..861d86858 --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Osv.Tests/Osv/OsvConnectorResumeWindowTests.cs @@ -0,0 +1,42 @@ +using StellaOps.Concelier.Connector.Osv; + +namespace StellaOps.Concelier.Connector.Osv.Tests; + +public sealed class OsvConnectorResumeWindowTests +{ + [Fact] + public void ComputeMinimumModified_WhenInitialBackfillWouldUnderflow_ClampsToMinValue() + { + var minimumModified = OsvConnector.ComputeMinimumModified( + existingLastModified: null, + now: DateTimeOffset.MinValue, + modifiedTolerance: TimeSpan.FromDays(2), + initialBackfill: TimeSpan.FromDays(30)); + + Assert.Equal(DateTimeOffset.MinValue, minimumModified); + } + + [Fact] + public void IsOutsideResumeWindow_WhenCurrentMaximumIsMinValue_DoesNotUnderflow() + { + var isOutsideWindow = OsvConnector.IsOutsideResumeWindow( + modified: DateTimeOffset.MinValue, + existingLastModified: null, + currentMaxModified: DateTimeOffset.MinValue, + modifiedTolerance: TimeSpan.FromDays(2)); + + Assert.False(isOutsideWindow); + } + + [Fact] + public void IsOutsideResumeWindow_WhenDocumentIsOlderThanLastModifiedTolerance_ReturnsTrue() + { + var isOutsideWindow = OsvConnector.IsOutsideResumeWindow( + modified: new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero), + existingLastModified: new DateTimeOffset(2026, 1, 10, 0, 0, 0, TimeSpan.Zero), + currentMaxModified: new DateTimeOffset(2026, 1, 10, 0, 0, 0, TimeSpan.Zero), + modifiedTolerance: TimeSpan.FromDays(1)); + + Assert.True(isOutsideWindow); + } +} diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Osv.Tests/Osv/OsvOptionsTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Osv.Tests/Osv/OsvOptionsTests.cs new file mode 100644 index 000000000..067445f4b --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Osv.Tests/Osv/OsvOptionsTests.cs @@ -0,0 +1,15 @@ +using StellaOps.Concelier.Connector.Osv.Configuration; + +namespace StellaOps.Concelier.Connector.Osv.Tests; + +public sealed class OsvOptionsTests +{ + [Fact] + public void DefaultEcosystems_UseCratesDotIoArchiveKey() + { + var options = new OsvOptions(); + + Assert.Contains("crates.io", options.Ecosystems); + Assert.DoesNotContain(options.Ecosystems, static ecosystem => string.Equals(ecosystem, "crates", StringComparison.OrdinalIgnoreCase)); + } +} diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Osv.Tests/TASKS.md b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Osv.Tests/TASKS.md index 5cc9638f9..b07926937 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Osv.Tests/TASKS.md +++ b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Osv.Tests/TASKS.md @@ -5,6 +5,7 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | Task ID | Status | Notes | | --- | --- | --- | +| CONN-ALIGN-003 | DOING | 2026-04-21 regression coverage: assert OSV defaults use `crates.io` so live fetches do not regress back to the dead archive key. | | AUDIT-0190-M | DONE | Revalidated 2026-01-06; findings recorded in audit report. | | AUDIT-0190-T | DONE | Revalidated 2026-01-06; findings recorded in audit report. | | AUDIT-0190-A | DONE | Waived (test project; revalidated 2026-01-06). | diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Adobe.Tests/TASKS.md b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Adobe.Tests/TASKS.md index c700fc7be..f869e4a4c 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Adobe.Tests/TASKS.md +++ b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Adobe.Tests/TASKS.md @@ -5,6 +5,7 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | Task ID | Status | Notes | | --- | --- | --- | +| CONN-ALIGN-007 | DONE | 2026-04-22: Updated Adobe connector fixture coverage to preserve deterministic canonical runtime aliasing under the host registration sprint. | | AUDIT-0198-M | DONE | Revalidated 2026-01-06; findings recorded in audit report. | | AUDIT-0198-T | DONE | Revalidated 2026-01-06; findings recorded in audit report. | | AUDIT-0198-A | DONE | Waived (test project; revalidated 2026-01-06). | diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/TASKS.md b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/TASKS.md index 841a12ad0..6d176fb69 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/TASKS.md +++ b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/TASKS.md @@ -5,6 +5,7 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | Task ID | Status | Notes | | --- | --- | --- | +| CONN-ALIGN-007 | DONE | 2026-04-22: Updated Chromium connector fixture coverage to preserve deterministic canonical runtime aliasing under the host registration sprint. | | AUDIT-0202-M | DONE | Revalidated 2026-01-06. | | AUDIT-0202-T | DONE | Revalidated 2026-01-06. | | AUDIT-0202-A | DONE | Waived (test project; revalidated 2026-01-06). | diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs index f76c1f79f..591123ccc 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs @@ -118,6 +118,35 @@ public sealed class OracleConnectorTests : IAsyncLifetime Assert.All(flags, flag => Assert.Equal("Oracle", flag.Vendor)); } + [Fact] + public async Task CalendarFetcher_FiltersNonAdvisoryLinks() + { + await using var provider = await BuildServiceProviderAsync(); + _handler.Clear(); + _handler.AddResponse(CalendarUri, () => new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + """ + + + CPU advisory + calendar index + product page + different host + + + """, + Encoding.UTF8, + "text/html"), + }); + + var calendarFetcher = provider.GetRequiredService(); + var discovered = await calendarFetcher.GetAdvisoryUrisAsync(CancellationToken.None); + + var advisory = Assert.Single(discovered); + Assert.Equal(AdvisoryOne, advisory); + } + [Fact] public async Task FetchAsync_IdempotentForUnchangedAdvisories() { @@ -351,4 +380,3 @@ public sealed class OracleConnectorTests : IAsyncLifetime } - diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/TASKS.md b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/TASKS.md index 1048616a2..57e571230 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/TASKS.md +++ b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/TASKS.md @@ -5,6 +5,7 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | Task ID | Status | Notes | | --- | --- | --- | +| CONN-ALIGN-003 | DOING | 2026-04-21 regression coverage: add Broadcom portal index + HTML detail fixtures so VMware stays runnable without the retired JSON feed. | | AUDIT-0210-M | DONE | Revalidated 2026-01-06. | | AUDIT-0210-T | DONE | Revalidated 2026-01-06. | | AUDIT-0210-A | DONE | Waived (test project; revalidated 2026-01-06). | diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/VmwareConnectorTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/VmwareConnectorTests.cs index 0f7d52dd0..d1969e950 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/VmwareConnectorTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/VmwareConnectorTests.cs @@ -41,6 +41,9 @@ public sealed class VmwareConnectorTests : IAsyncLifetime private static readonly Uri DetailOne = new("https://vmware.example/api/vmsa/VMSA-2024-0001.json"); private static readonly Uri DetailTwo = new("https://vmware.example/api/vmsa/VMSA-2024-0002.json"); private static readonly Uri DetailThree = new("https://vmware.example/api/vmsa/VMSA-2024-0003.json"); + private static readonly Uri BroadcomIndexUri = new("https://support.broadcom.com/web/ecx/security-advisory?segment=VC"); + private static readonly Uri BroadcomIndexApiUri = new("https://support.broadcom.com/web/ecx/security-advisory/-/securityadvisory/getSecurityAdvisoryList"); + private static readonly Uri BroadcomDetailUri = new("https://support.broadcom.com/web/ecx/support-content-notification/-/external/content/SecurityAdvisories/0/36947"); public VmwareConnectorTests(ConcelierPostgresFixture fixture, ITestOutputHelper output) { @@ -156,6 +159,34 @@ public sealed class VmwareConnectorTests : IAsyncLifetime Assert.Equal(new[] { 1, 1, 2 }, affectedCounts); } + [Fact] + public async Task FetchParseMap_BroadcomPortalMode_UsesPortalIndexAndHtmlDetail() + { + await using var provider = await BuildServiceProviderAsync(BroadcomIndexUri); + SeedBroadcomPortalResponses(); + + var connector = provider.GetRequiredService(); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + var advisory = Assert.Single(advisories); + + Assert.Equal("VMSA-2026-0001", advisory.AdvisoryKey); + Assert.Contains("CVE-2026-22719", advisory.Aliases); + Assert.Contains("CVE-2026-22720", advisory.Aliases); + Assert.Contains("CVE-2026-22721", advisory.Aliases); + Assert.NotEmpty(advisory.AffectedPackages); + Assert.Contains(advisory.AffectedPackages, package => string.Equals(package.Identifier, "VMware Aria Operations", StringComparison.Ordinal)); + Assert.Contains(advisory.References, reference => string.Equals(reference.Url, BroadcomDetailUri.ToString(), StringComparison.OrdinalIgnoreCase)); + + Assert.Contains(_handler.Requests, request => request.Method == HttpMethod.Post && request.Uri == BroadcomIndexApiUri); + Assert.Contains(_handler.Requests, request => request.Method == HttpMethod.Get && request.Uri == BroadcomDetailUri); + } + public ValueTask InitializeAsync() => ValueTask.CompletedTask; public ValueTask DisposeAsync() @@ -164,7 +195,7 @@ public sealed class VmwareConnectorTests : IAsyncLifetime return ValueTask.CompletedTask; } - private async Task BuildServiceProviderAsync() + private async Task BuildServiceProviderAsync(Uri? indexUri = null) { await _fixture.TruncateAllTablesAsync(); _handler.Clear(); @@ -184,7 +215,7 @@ public sealed class VmwareConnectorTests : IAsyncLifetime services.AddSourceCommon(); services.AddVmwareConnector(opts => { - opts.IndexUri = IndexUri; + opts.IndexUri = indexUri ?? IndexUri; opts.InitialBackfill = TimeSpan.FromDays(30); opts.ModifiedTolerance = TimeSpan.FromMinutes(5); opts.MaxAdvisoriesPerFetch = 10; @@ -214,6 +245,94 @@ public sealed class VmwareConnectorTests : IAsyncLifetime _handler.AddJsonResponse(DetailThree, ReadFixture("vmware-detail-vmsa-2024-0003.json")); } + private void SeedBroadcomPortalResponses() + { + _handler.AddTextResponse(BroadcomIndexUri, """ + + + VMware Security Advisories + + """, contentType: "text/html"); + + _handler.AddResponse(HttpMethod.Post, BroadcomIndexApiUri, request => + { + Assert.True(request.Headers.TryGetValues("X-CSRF-Token", out var tokenValues)); + Assert.Contains("portal-token", tokenValues); + Assert.Equal(BroadcomIndexUri, request.Headers.Referrer); + + var body = request.Content?.ReadAsStringAsync().GetAwaiter().GetResult() ?? string.Empty; + Assert.Contains("\"pageNumber\":0", body, StringComparison.Ordinal); + + return new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new StringContent(""" + { + "success": true, + "data": { + "list": [ + { + "documentId": "doc-36947", + "notificationId": "36947", + "published": "24 February 2026", + "updated": "2026-03-11 07:07:22.338", + "title": "VMSA-2026-0001: VMware Aria Operations updates address multiple vulnerabilities (CVE-2026-22719, CVE-2026-22720 and CVE-2026-22721)", + "notificationUrl": "https://support.broadcom.com/web/ecx/support-content-notification/-/external/content/SecurityAdvisories/0/36947" + } + ] + } + } + """, System.Text.Encoding.UTF8, "application/json"), + }; + }); + + _handler.AddResponse(HttpMethod.Post, BroadcomIndexApiUri, request => + { + Assert.True(request.Headers.TryGetValues("X-CSRF-Token", out var tokenValues)); + Assert.Contains("portal-token", tokenValues); + Assert.Equal(BroadcomIndexUri, request.Headers.Referrer); + + var body = request.Content?.ReadAsStringAsync().GetAwaiter().GetResult() ?? string.Empty; + Assert.Contains("\"pageNumber\":1", body, StringComparison.Ordinal); + + return new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new StringContent(""" + { + "success": true, + "data": { + "list": [] + } + } + """, System.Text.Encoding.UTF8, "application/json"), + }; + }); + + _handler.AddTextResponse(BroadcomDetailUri, """ + + +

VMSA-2026-0001: VMware Aria Operations updates address multiple vulnerabilities (CVE-2026-22719, CVE-2026-22720 and CVE-2026-22721)

+
Initial Publication Date
+
24 February 2026
+
Last Updated
+
11 March 2026
+
Advisory ID:
+
VMSA-2026-0001.1
+
Synopsis: VMware Aria Operations updates address multiple vulnerabilities (CVE-2026-22719, CVE-2026-22720 and CVE-2026-22721)
+
CVE(s) CVE-2026-22719, CVE-2026-22720, CVE-2026-22721
+

## 1. Impacted Products

+
    +
  • VMware Aria Operations
    • +
  • VMware Cloud Foundation
    • +
+

## 2. Introduction

+

Multiple vulnerabilities in VMware Aria Operations were privately reported to Broadcom. Patches and workarounds are available.

+ KB430349 + CVE-2026-22719 + + + """, contentType: "text/html"); + } + private static string ReadFixture(string name) { var primary = Path.Combine(AppContext.BaseDirectory, "Vmware", "Fixtures", name); @@ -277,5 +396,3 @@ public sealed class VmwareConnectorTests : IAsyncLifetime } } - - diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/SourceKeyAliasesTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/SourceKeyAliasesTests.cs new file mode 100644 index 000000000..43356f0ca --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/SourceKeyAliasesTests.cs @@ -0,0 +1,54 @@ +using StellaOps.Concelier.Core.Sources; +using StellaOps.TestKit; + +namespace StellaOps.Concelier.Core.Tests; + +public sealed class SourceKeyAliasesTests +{ + [Trait("Category", TestCategories.Unit)] + [Theory] + [InlineData("distro-alpine", "alpine")] + [InlineData("distro-debian", "debian")] + [InlineData("ics-cisa", "us-cert")] + [InlineData("ics-kaspersky", "kaspersky-ics")] + [InlineData("ru-bdu", "fstec-bdu")] + [InlineData("ru-nkcki", "nkcki")] + [InlineData("vndr-adobe", "adobe")] + [InlineData("vndr-apple", "apple")] + [InlineData("vndr-chromium", "chromium")] + public void Normalize_MapsLegacyConnectorKeysToCanonicalCatalogIds(string sourceKey, string expected) + { + var actual = SourceKeyAliases.Normalize(sourceKey); + + Assert.Equal(expected, actual); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public void GetEquivalentKeys_IncludesCanonicalAndAliasWithoutDuplicates() + { + var keys = SourceKeyAliases.GetEquivalentKeys("distro-debian"); + + Assert.Equal(new[] { "debian", "distro-debian" }, keys); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public void GetEquivalentKeys_IncludesNewIcsAliases() + { + var keys = SourceKeyAliases.GetEquivalentKeys("us-cert"); + + Assert.Equal(new[] { "us-cert", "ics-cisa" }, keys); + } + + [Trait("Category", TestCategories.Unit)] + [Theory] + [InlineData("adobe", new[] { "adobe", "vndr-adobe" })] + [InlineData("chromium", new[] { "chromium", "vndr-chromium" })] + public void GetEquivalentKeys_IncludesVendorAliases(string sourceKey, string[] expected) + { + var keys = SourceKeyAliases.GetEquivalentKeys(sourceKey); + + Assert.Equal(expected, keys); + } +} diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/TASKS.md b/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/TASKS.md index faa431f7c..83601e5fd 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/TASKS.md +++ b/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/TASKS.md @@ -9,3 +9,5 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | AUDIT-0212-T | DONE | Revalidated 2026-01-06. | | AUDIT-0212-A | DONE | Waived (test project; revalidated 2026-01-06). | | REMED-08 | DONE | Added ingestion telemetry metric tag tests; `dotnet test src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/StellaOps.Concelier.Core.Tests.csproj` passed (2026-02-03). | +| CONN-ALIGN-003 | DONE | Added `SourceKeyAliases` regression coverage so advisory-source alias rows collapse to the canonical operator-facing IDs in runtime APIs/UI (2026-04-21). | +| CONN-ALIGN-007 | DONE | 2026-04-22: Extended alias normalization coverage for the canonical `adobe` and `chromium` runtime source IDs. | diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/AdvisorySourceReadRepositoryTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/AdvisorySourceReadRepositoryTests.cs new file mode 100644 index 000000000..42008ad72 --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/AdvisorySourceReadRepositoryTests.cs @@ -0,0 +1,181 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Persistence.Postgres; +using StellaOps.Concelier.Persistence.Postgres.Models; +using StellaOps.Concelier.Persistence.Postgres.Repositories; +using StellaOps.TestKit; +using Xunit; + +namespace StellaOps.Concelier.Persistence.Tests; + +[Collection(ConcelierPostgresCollection.Name)] +public sealed class AdvisorySourceReadRepositoryTests : IAsyncLifetime +{ + private readonly ConcelierPostgresFixture _fixture; + private readonly ConcelierDataSource _dataSource; + private readonly SourceRepository _sourceRepository; + private readonly SourceStateRepository _sourceStateRepository; + private readonly AdvisoryCanonicalRepository _canonicalRepository; + private readonly AdvisorySourceReadRepository _repository; + + public AdvisorySourceReadRepositoryTests(ConcelierPostgresFixture fixture) + { + _fixture = fixture; + + var options = fixture.CreateOptions(); + _dataSource = new ConcelierDataSource(Options.Create(options), NullLogger.Instance); + _sourceRepository = new SourceRepository(_dataSource, NullLogger.Instance); + _sourceStateRepository = new SourceStateRepository(_dataSource, NullLogger.Instance); + _canonicalRepository = new AdvisoryCanonicalRepository(_dataSource, NullLogger.Instance); + _repository = new AdvisorySourceReadRepository( + _dataSource, + NullLogger.Instance, + new StaticTimeProvider(new DateTimeOffset(2026, 04, 21, 10, 00, 00, TimeSpan.Zero))); + } + + public ValueTask InitializeAsync() => new(_fixture.TruncateAllTablesAsync()); + public async ValueTask DisposeAsync() => await _dataSource.DisposeAsync(); + + [Trait("Category", TestCategories.Integration)] + [Fact] + public async Task ListAsync_ShouldCountDistinctSourceAdvisoriesAcrossCanonicalFanOut() + { + var source = await _sourceRepository.UpsertAsync(CreateSource("redhat", metadata: """{"signature_status":"verified"}""")); + await _sourceStateRepository.UpsertAsync(CreateState( + source.Id, + lastSuccessAt: new DateTimeOffset(2026, 04, 21, 09, 55, 00, TimeSpan.Zero), + metadata: """{"signature_failure_count":"3"}""")); + + var canonicalA = await InsertCanonicalAsync("CVE-2026-0001", "pkg:rpm/rhel/kernel@1.0.0"); + var canonicalB = await InsertCanonicalAsync("CVE-2026-0002", "pkg:rpm/rhel/kernel-core@1.0.0"); + var canonicalC = await InsertCanonicalAsync("CVE-2026-0003", "pkg:rpm/rhel/kernel-modules@1.0.0"); + + await _canonicalRepository.AddSourceEdgeAsync(CreateSourceEdge( + canonicalA.Id, + source.Id, + "RHSA-2026:0001", + "sha256:source-doc-1", + """{"payloadType":"application/vnd.dsse+json","payload":"e30=","signatures":[{"keyid":"sig-1","sig":"abc"}]}""")); + await _canonicalRepository.AddSourceEdgeAsync(CreateSourceEdge( + canonicalB.Id, + source.Id, + "RHSA-2026:0001", + "sha256:source-doc-2")); + await _canonicalRepository.AddSourceEdgeAsync(CreateSourceEdge( + canonicalC.Id, + source.Id, + "RHSA-2026:0002", + "sha256:source-doc-3")); + + var records = await _repository.ListAsync(includeDisabled: true); + + var record = records.Should().ContainSingle(r => r.SourceId == source.Id).Subject; + record.SourceDocumentCount.Should().Be(2); + record.CanonicalAdvisoryCount.Should().Be(3); + record.CveCount.Should().Be(3); + record.VexDocumentCount.Should().Be(2); + record.TotalAdvisories.Should().Be(2); + record.SignedAdvisories.Should().Be(1); + record.UnsignedAdvisories.Should().Be(1); + record.SignatureFailureCount.Should().Be(3); + record.SignatureStatus.Should().Be("verified"); + record.FreshnessStatus.Should().Be("healthy"); + } + + [Trait("Category", TestCategories.Integration)] + [Fact] + public async Task GetBySourceIdAsync_ShouldReturnZeroCountsWhenNoSourceDocumentsExist() + { + var source = await _sourceRepository.UpsertAsync(CreateSource("debian")); + await _sourceStateRepository.UpsertAsync(CreateState( + source.Id, + lastSuccessAt: new DateTimeOffset(2026, 04, 21, 09, 30, 00, TimeSpan.Zero))); + + var record = await _repository.GetBySourceIdAsync(source.Id); + + record.Should().NotBeNull(); + record!.SourceDocumentCount.Should().Be(0); + record.CanonicalAdvisoryCount.Should().Be(0); + record.CveCount.Should().Be(0); + record.VexDocumentCount.Should().Be(0); + record!.TotalAdvisories.Should().Be(0); + record.SignedAdvisories.Should().Be(0); + record.UnsignedAdvisories.Should().Be(0); + record.FreshnessStatus.Should().Be("healthy"); + } + + private async Task InsertCanonicalAsync(string cve, string affectsKey) + { + var entity = new AdvisoryCanonicalEntity + { + Id = Guid.NewGuid(), + Cve = cve, + AffectsKey = affectsKey, + MergeHash = $"sha256:{Guid.NewGuid():N}", + Weakness = [], + ExploitKnown = false + }; + + var id = await _canonicalRepository.UpsertAsync(entity); + return (await _canonicalRepository.GetByIdAsync(id))!; + } + + private static SourceEntity CreateSource(string key, string? metadata = null) => new() + { + Id = Guid.NewGuid(), + Key = key, + Name = $"Source {key}", + SourceType = key, + Url = $"https://{key}.example.test/feed", + Priority = 100, + Enabled = true, + Config = "{}", + Metadata = metadata ?? "{}", + CreatedAt = DateTimeOffset.UtcNow, + UpdatedAt = DateTimeOffset.UtcNow + }; + + private static SourceStateEntity CreateState(Guid sourceId, DateTimeOffset lastSuccessAt, string? metadata = null) => new() + { + Id = Guid.NewGuid(), + SourceId = sourceId, + LastSyncAt = lastSuccessAt, + LastSuccessAt = lastSuccessAt, + Cursor = "{}", + ErrorCount = 0, + SyncCount = 1, + Metadata = metadata ?? "{}", + UpdatedAt = lastSuccessAt + }; + + private static AdvisorySourceEdgeEntity CreateSourceEdge( + Guid canonicalId, + Guid sourceId, + string sourceAdvisoryId, + string sourceDocHash, + string? dsseEnvelope = null) => new() + { + Id = Guid.NewGuid(), + CanonicalId = canonicalId, + SourceId = sourceId, + SourceAdvisoryId = sourceAdvisoryId, + SourceDocHash = sourceDocHash, + VendorStatus = "affected", + PrecedenceRank = 10, + DsseEnvelope = dsseEnvelope, + FetchedAt = new DateTimeOffset(2026, 04, 21, 09, 50, 00, TimeSpan.Zero) + }; + + private sealed class StaticTimeProvider : TimeProvider + { + private readonly DateTimeOffset _utcNow; + + public StaticTimeProvider(DateTimeOffset utcNow) + { + _utcNow = utcNow; + } + + public override DateTimeOffset GetUtcNow() => _utcNow; + } +} diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/ConcelierInfrastructureRegistrationTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/ConcelierInfrastructureRegistrationTests.cs index 81f9c856b..5f776fc88 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/ConcelierInfrastructureRegistrationTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/ConcelierInfrastructureRegistrationTests.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Hosting; using StellaOps.Concelier.Core.Jobs; using StellaOps.Concelier.Core.Linksets; using StellaOps.Concelier.Core.Observations; +using StellaOps.Concelier.Core.Orchestration; using StellaOps.Concelier.Core.Signals; using StellaOps.Concelier.Persistence.Postgres; using StellaOps.Concelier.Persistence.Postgres.Repositories; @@ -64,7 +65,6 @@ public sealed class ConcelierInfrastructureRegistrationTests { var services = new ServiceCollection(); services.AddLogging(); - services.AddSingleton(TimeProvider.System); services.AddConcelierSignalsServices(); services.AddConcelierPostgresStorage(options => { @@ -89,4 +89,35 @@ public sealed class ConcelierInfrastructureRegistrationTests await provider.DisposeAsync(); } } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task AddConcelierPostgresStorage_RegistersDurableJobAndOrchestratorRuntimeServices() + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddConcelierPostgresStorage(options => + { + options.ConnectionString = "Host=postgres;Database=stellaops;Username=postgres;Password=postgres"; + options.SchemaName = "vuln"; + }); + services.AddJobScheduler(); + services.AddConcelierOrchestrationServices(); + + var provider = services.BuildServiceProvider(new ServiceProviderOptions + { + ValidateScopes = true + }); + + try + { + provider.GetRequiredService().Should().BeOfType(); + provider.GetRequiredService().Should().BeOfType(); + provider.GetRequiredService().Should().BeOfType(); + } + finally + { + await provider.DisposeAsync(); + } + } } diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/ConcelierPostgresFixture.cs b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/ConcelierPostgresFixture.cs index 0733738e9..b14414c08 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/ConcelierPostgresFixture.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/ConcelierPostgresFixture.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Logging; using System.Reflection; using StellaOps.Concelier.Persistence.Postgres; using StellaOps.Infrastructure.Postgres.Testing; @@ -17,6 +18,10 @@ public sealed class ConcelierPostgresFixture : PostgresIntegrationFixture, IColl protected override string GetModuleName() => "Concelier"; + protected override PostgresFixture CreateFixtureInstance(string connectionString, string moduleName, ILogger logger) + // Concelier persistence currently assumes the canonical runtime schema for embedded SQL migrations. + => new(connectionString, ConcelierDataSource.DefaultSchemaName, logger); + public PostgresOptions CreateOptions() { var options = Fixture.CreateOptions(); diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/PostgresAdvisoryObservationStoreTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/PostgresAdvisoryObservationStoreTests.cs index e34f99c7f..dea46e4f2 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/PostgresAdvisoryObservationStoreTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/PostgresAdvisoryObservationStoreTests.cs @@ -80,7 +80,7 @@ public sealed class PostgresAdvisoryObservationStoreTests : IAsyncLifetime filtered[0].ObservationId.Should().Be("obs-osv-001"); filtered[0].Tenant.Should().Be("tenant-a"); symbols.Should().HaveCount(2); - symbols.Select(symbol => symbol.Symbol).Should().BeEquivalentTo("dangerousCall", "Run"); + symbols.Select(symbol => symbol.Symbol).Should().BeEquivalentTo(new[] { "dangerousCall", "Run" }); } [Trait("Category", TestCategories.Unit)] diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/PostgresAdvisoryStoreTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/PostgresAdvisoryStoreTests.cs new file mode 100644 index 000000000..81ffc1306 --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/PostgresAdvisoryStoreTests.cs @@ -0,0 +1,91 @@ +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Persistence.Postgres; +using StellaOps.Concelier.Persistence.Postgres.Repositories; +using StellaOps.TestKit; +using Xunit; +using AdvisoryContracts = StellaOps.Concelier.Storage.Advisories; + +namespace StellaOps.Concelier.Persistence.Tests; + +[Collection(ConcelierPostgresCollection.Name)] +public sealed class PostgresAdvisoryStoreTests : IAsyncLifetime +{ + private readonly ConcelierPostgresFixture _fixture; + private ServiceProvider _provider = null!; + + public PostgresAdvisoryStoreTests(ConcelierPostgresFixture fixture) + { + _fixture = fixture; + } + + public async ValueTask InitializeAsync() + { + await _fixture.TruncateAllTablesAsync(); + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddConcelierPostgresStorage(options => + { + options.ConnectionString = _fixture.ConnectionString; + options.SchemaName = _fixture.SchemaName; + options.CommandTimeoutSeconds = 5; + }); + + _provider = services.BuildServiceProvider(new ServiceProviderOptions + { + ValidateScopes = true + }); + } + + public async ValueTask DisposeAsync() + { + if (_provider is not null) + { + await _provider.DisposeAsync(); + } + } + + [Trait("Category", TestCategories.Integration)] + [Fact] + public async Task LegacyAdvisoryStoreUpsertAsync_ResolvesSourceIdFromProvenance() + { + var recordedAt = new DateTimeOffset(2026, 4, 22, 9, 0, 0, TimeSpan.Zero); + var advisory = new Advisory( + advisoryKey: "kev/cve-2026-9999", + title: "KEV regression test advisory", + summary: "Regression coverage for advisory source binding.", + language: "en", + published: recordedAt, + modified: recordedAt, + severity: "high", + exploitKnown: true, + aliases: ["CVE-2026-9999"], + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: + [ + new AdvisoryProvenance( + source: "kev", + kind: "mapping", + value: "fixture", + recordedAt: recordedAt) + ]); + + using var scope = _provider.CreateScope(); + var advisoryStore = scope.ServiceProvider.GetRequiredService(); + var advisoryRepository = scope.ServiceProvider.GetRequiredService(); + var sourceRepository = scope.ServiceProvider.GetRequiredService(); + + await advisoryStore.UpsertAsync(advisory, CancellationToken.None); + + var source = await sourceRepository.GetByKeyAsync("kev", CancellationToken.None); + source.Should().NotBeNull(); + + var persisted = await advisoryRepository.GetByKeyAsync(advisory.AdvisoryKey, CancellationToken.None); + persisted.Should().NotBeNull(); + persisted!.SourceId.Should().Be(source!.Id); + } +} diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/PostgresJobStoreTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/PostgresJobStoreTests.cs new file mode 100644 index 000000000..873961987 --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/PostgresJobStoreTests.cs @@ -0,0 +1,227 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Persistence.Postgres; +using StellaOps.TestKit; +using Xunit; + +namespace StellaOps.Concelier.Persistence.Tests; + +[Collection(ConcelierPostgresCollection.Name)] +public sealed class PostgresJobStoreTests : IAsyncLifetime +{ + private readonly ConcelierPostgresFixture _fixture; + private readonly ConcelierDataSource _dataSource; + private readonly StellaOps.Concelier.Persistence.Postgres.Repositories.PostgresJobStore _store; + private readonly StellaOps.Concelier.Persistence.Postgres.Repositories.PostgresLeaseStore _leaseStore; + + public PostgresJobStoreTests(ConcelierPostgresFixture fixture) + { + _fixture = fixture; + _dataSource = CreateDataSource(); + _store = new StellaOps.Concelier.Persistence.Postgres.Repositories.PostgresJobStore( + _dataSource, + NullLogger.Instance); + _leaseStore = new StellaOps.Concelier.Persistence.Postgres.Repositories.PostgresLeaseStore( + _dataSource, + NullLogger.Instance); + } + + public ValueTask InitializeAsync() => new(_fixture.TruncateAllTablesAsync()); + + public async ValueTask DisposeAsync() + { + await _dataSource.DisposeAsync(); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task CreateAsync_PersistsRunAcrossStoreInstancesAndRoundTripsParameters() + { + var request = new JobRunCreateRequest( + "sync:nvd", + "api", + new Dictionary(StringComparer.Ordinal) + { + ["tenant"] = "tenant-a", + ["depth"] = 2L, + ["flags"] = new object?[] { "full", true } + }, + "abc123", + TimeSpan.FromMinutes(10), + TimeSpan.FromMinutes(3), + DateTimeOffset.Parse("2026-04-19T12:00:00Z")); + + var created = await _store.CreateAsync(request, TestContext.Current.CancellationToken); + + await using var restartedDataSource = CreateDataSource(); + var restartedStore = new StellaOps.Concelier.Persistence.Postgres.Repositories.PostgresJobStore( + restartedDataSource, + NullLogger.Instance); + var persisted = await restartedStore.FindAsync(created.RunId, TestContext.Current.CancellationToken); + + persisted.Should().NotBeNull(); + persisted!.RunId.Should().Be(created.RunId); + persisted.Kind.Should().Be("sync:nvd"); + persisted.Trigger.Should().Be("api"); + persisted.Status.Should().Be(JobRunStatus.Pending); + persisted.ParametersHash.Should().Be("abc123"); + persisted.Timeout.Should().Be(TimeSpan.FromMinutes(10)); + persisted.LeaseDuration.Should().Be(TimeSpan.FromMinutes(3)); + persisted.Parameters["tenant"].Should().Be("tenant-a"); + persisted.Parameters["depth"].Should().Be(2L); + persisted.Parameters["flags"].Should().BeEquivalentTo(new object?[] { "full", true }); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task TryStartAsync_AndTryCompleteAsync_UpdateStatusAndActiveRuns() + { + var created = await _store.CreateAsync( + CreateRequest("sync:ghsa", "2026-04-19T12:10:00Z"), + TestContext.Current.CancellationToken); + + var startedAt = DateTimeOffset.Parse("2026-04-19T12:11:00Z"); + var started = await _store.TryStartAsync(created.RunId, startedAt, TestContext.Current.CancellationToken); + var leaseAcquiredAt = DateTimeOffset.UtcNow; + await _leaseStore.TryAcquireAsync( + "job:sync:ghsa", + "node-a", + TimeSpan.FromMinutes(5), + leaseAcquiredAt, + TestContext.Current.CancellationToken); + var active = await _store.GetActiveRunsAsync(TestContext.Current.CancellationToken); + + started.Should().NotBeNull(); + started!.Status.Should().Be(JobRunStatus.Running); + started.StartedAt.Should().Be(startedAt); + active.Select(run => run.RunId).Should().Contain(created.RunId); + + var completion = new JobRunCompletion( + JobRunStatus.Succeeded, + DateTimeOffset.Parse("2026-04-19T12:12:00Z"), + null); + var completed = await _store.TryCompleteAsync(created.RunId, completion, TestContext.Current.CancellationToken); + var activeAfterComplete = await _store.GetActiveRunsAsync(TestContext.Current.CancellationToken); + + completed.Should().NotBeNull(); + completed!.Status.Should().Be(JobRunStatus.Succeeded); + completed.CompletedAt.Should().Be(completion.CompletedAt); + activeAfterComplete.Should().BeEmpty(); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task GetRecentRunsAsync_AndGetLastRunsAsync_FilterAndOrderByCreatedAt() + { + var nvdOlder = await _store.CreateAsync( + CreateRequest("sync:nvd", "2026-04-19T12:20:00Z"), + TestContext.Current.CancellationToken); + var ghsa = await _store.CreateAsync( + CreateRequest("sync:ghsa", "2026-04-19T12:21:00Z"), + TestContext.Current.CancellationToken); + var nvdNewer = await _store.CreateAsync( + CreateRequest("sync:nvd", "2026-04-19T12:22:00Z"), + TestContext.Current.CancellationToken); + + var recentNvd = await _store.GetRecentRunsAsync("sync:nvd", 10, TestContext.Current.CancellationToken); + var lastNvd = await _store.GetLastRunAsync("sync:nvd", TestContext.Current.CancellationToken); + var lastRuns = await _store.GetLastRunsAsync(["sync:nvd", "sync:ghsa"], TestContext.Current.CancellationToken); + + recentNvd.Select(run => run.RunId).Should().ContainInOrder(nvdNewer.RunId, nvdOlder.RunId); + lastNvd.Should().NotBeNull(); + lastNvd!.RunId.Should().Be(nvdNewer.RunId); + lastRuns["sync:nvd"].RunId.Should().Be(nvdNewer.RunId); + lastRuns["sync:ghsa"].RunId.Should().Be(ghsa.RunId); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task GetActiveRunsAsync_OnlyReturnsLeaseBackedActiveRuns() + { + var staleCreated = await _store.CreateAsync( + CreateRequest("sync:cert-fr", "2026-04-19T12:30:00Z"), + TestContext.Current.CancellationToken); + await _store.TryStartAsync( + staleCreated.RunId, + DateTimeOffset.Parse("2026-04-19T12:31:00Z"), + TestContext.Current.CancellationToken); + + var liveNow = DateTimeOffset.UtcNow; + var liveCreated = await _store.CreateAsync( + CreateRequest("sync:oracle", liveNow.AddSeconds(-30).ToString("O")), + TestContext.Current.CancellationToken); + await _store.TryStartAsync( + liveCreated.RunId, + liveNow.AddSeconds(-20), + TestContext.Current.CancellationToken); + await _leaseStore.TryAcquireAsync( + "job:sync:oracle", + "node-a", + TimeSpan.FromMinutes(5), + liveNow, + TestContext.Current.CancellationToken); + + var active = await _store.GetActiveRunsAsync(TestContext.Current.CancellationToken); + + active.Select(run => run.RunId).Should().Contain(liveCreated.RunId); + active.Select(run => run.RunId).Should().NotContain(staleCreated.RunId); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task GetActiveRunsAsync_ReturnsNewestLeaseBackedRunPerKind() + { + var older = await _store.CreateAsync( + CreateRequest("sync:osv", "2026-04-19T12:40:00Z"), + TestContext.Current.CancellationToken); + await _store.TryStartAsync( + older.RunId, + DateTimeOffset.Parse("2026-04-19T12:41:00Z"), + TestContext.Current.CancellationToken); + + var recentNow = DateTimeOffset.UtcNow; + var newer = await _store.CreateAsync( + CreateRequest("sync:osv", recentNow.AddSeconds(-10).ToString("O")), + TestContext.Current.CancellationToken); + await _store.TryStartAsync( + newer.RunId, + recentNow.AddSeconds(-5), + TestContext.Current.CancellationToken); + await _leaseStore.TryAcquireAsync( + "job:sync:osv", + "node-a", + TimeSpan.FromMinutes(5), + recentNow, + TestContext.Current.CancellationToken); + + var active = await _store.GetActiveRunsAsync(TestContext.Current.CancellationToken); + var osvRuns = active.Where(run => run.Kind == "sync:osv").ToArray(); + + osvRuns.Should().ContainSingle(); + osvRuns[0].RunId.Should().Be(newer.RunId); + } + + private ConcelierDataSource CreateDataSource() + { + var options = _fixture.CreateOptions(); + return new ConcelierDataSource(Options.Create(options), NullLogger.Instance); + } + + private static JobRunCreateRequest CreateRequest(string kind, string createdAt) + { + return new JobRunCreateRequest( + kind, + "scheduler", + new Dictionary(StringComparer.Ordinal) + { + ["source"] = kind, + ["batch"] = DateTimeOffset.Parse(createdAt).ToUnixTimeSeconds() + }, + $"hash-{kind}-{createdAt}", + TimeSpan.FromMinutes(5), + TimeSpan.FromMinutes(2), + DateTimeOffset.Parse(createdAt)); + } +} diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/PostgresOrchestratorRegistryStoreTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/PostgresOrchestratorRegistryStoreTests.cs new file mode 100644 index 000000000..78ef9c05e --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/PostgresOrchestratorRegistryStoreTests.cs @@ -0,0 +1,200 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Core.Orchestration; +using StellaOps.Concelier.Persistence.Postgres; +using StellaOps.TestKit; +using Xunit; + +namespace StellaOps.Concelier.Persistence.Tests; + +[Collection(ConcelierPostgresCollection.Name)] +public sealed class PostgresOrchestratorRegistryStoreTests : IAsyncLifetime +{ + private readonly ConcelierPostgresFixture _fixture; + private readonly ConcelierDataSource _dataSource; + private readonly StellaOps.Concelier.Persistence.Postgres.Repositories.PostgresOrchestratorRegistryStore _store; + + public PostgresOrchestratorRegistryStoreTests(ConcelierPostgresFixture fixture) + { + _fixture = fixture; + _dataSource = CreateDataSource(); + _store = new StellaOps.Concelier.Persistence.Postgres.Repositories.PostgresOrchestratorRegistryStore( + _dataSource, + TimeProvider.System, + NullLogger.Instance); + } + + public ValueTask InitializeAsync() => new(_fixture.TruncateAllTablesAsync()); + + public async ValueTask DisposeAsync() + { + await _dataSource.DisposeAsync(); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task UpsertAsync_RoundTripsRegistryAndListsByConnectorId() + { + await _store.UpsertAsync(CreateRegistryRecord("tenant-a", "zzz-connector"), TestContext.Current.CancellationToken); + await _store.UpsertAsync(CreateRegistryRecord("tenant-a", "aaa-connector", source: "osv"), TestContext.Current.CancellationToken); + + var record = await _store.GetAsync("tenant-a", "aaa-connector", TestContext.Current.CancellationToken); + var records = await _store.ListAsync("tenant-a", TestContext.Current.CancellationToken); + + record.Should().NotBeNull(); + record!.Source.Should().Be("osv"); + record.Schedule.Cron.Should().Be("0 * * * *"); + record.RatePolicy.Rpm.Should().Be(120); + record.EgressGuard.Allowlist.Should().BeEquivalentTo(["mirror.example"]); + records.Select(item => item.ConnectorId).Should().ContainInOrder("aaa-connector", "zzz-connector"); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task AppendHeartbeatAsync_ReturnsLatestHeartbeatAcrossStoreInstances() + { + var runId = Guid.Parse("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"); + await _store.AppendHeartbeatAsync( + new OrchestratorHeartbeatRecord( + "tenant-a", + "nvd", + runId, + 1, + OrchestratorHeartbeatStatus.Starting, + 5, + 1, + null, + null, + null, + null, + DateTimeOffset.Parse("2026-04-19T13:00:00Z")), + TestContext.Current.CancellationToken); + await _store.AppendHeartbeatAsync( + new OrchestratorHeartbeatRecord( + "tenant-a", + "nvd", + runId, + 3, + OrchestratorHeartbeatStatus.Succeeded, + 100, + 0, + "sha256:artifact", + "normalized", + null, + null, + DateTimeOffset.Parse("2026-04-19T13:02:00Z")), + TestContext.Current.CancellationToken); + + await using var restartedDataSource = CreateDataSource(); + var restartedStore = new StellaOps.Concelier.Persistence.Postgres.Repositories.PostgresOrchestratorRegistryStore( + restartedDataSource, + TimeProvider.System, + NullLogger.Instance); + + var latest = await restartedStore.GetLatestHeartbeatAsync("tenant-a", "nvd", runId, TestContext.Current.CancellationToken); + + latest.Should().NotBeNull(); + latest!.Sequence.Should().Be(3); + latest.Status.Should().Be(OrchestratorHeartbeatStatus.Succeeded); + latest.LastArtifactHash.Should().Be("sha256:artifact"); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task GetPendingCommandsAsync_FiltersExpiredCommands_AndManifestRoundTrips() + { + var runId = Guid.Parse("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"); + await _store.EnqueueCommandAsync( + new OrchestratorCommandRecord( + "tenant-a", + "nvd", + runId, + 1, + OrchestratorCommandKind.Pause, + null, + null, + DateTimeOffset.Parse("2026-04-19T13:10:00Z"), + DateTimeOffset.Parse("2000-01-01T00:00:00Z")), + TestContext.Current.CancellationToken); + await _store.EnqueueCommandAsync( + new OrchestratorCommandRecord( + "tenant-a", + "nvd", + runId, + 2, + OrchestratorCommandKind.Throttle, + new OrchestratorThrottleOverride(60, 10, 30, DateTimeOffset.Parse("2100-01-01T00:10:00Z")), + null, + DateTimeOffset.Parse("2026-04-19T13:11:00Z"), + DateTimeOffset.Parse("2100-01-01T00:05:00Z")), + TestContext.Current.CancellationToken); + await _store.EnqueueCommandAsync( + new OrchestratorCommandRecord( + "tenant-a", + "nvd", + runId, + 3, + OrchestratorCommandKind.Backfill, + null, + new OrchestratorBackfillRange("cursor-a", "cursor-z"), + DateTimeOffset.Parse("2026-04-19T13:12:00Z"), + DateTimeOffset.Parse("2100-01-01T00:06:00Z")), + TestContext.Current.CancellationToken); + + var allPendingCommands = await _store.GetPendingCommandsAsync("tenant-a", "nvd", runId, null, TestContext.Current.CancellationToken); + var commands = await _store.GetPendingCommandsAsync("tenant-a", "nvd", runId, 1, TestContext.Current.CancellationToken); + + allPendingCommands.Select(command => command.Sequence).Should().ContainInOrder(2L, 3L); + commands.Select(command => command.Sequence).Should().ContainInOrder(2L, 3L); + commands[0].Throttle.Should().NotBeNull(); + commands[0].Throttle!.Rpm.Should().Be(60); + commands[1].Backfill.Should().NotBeNull(); + commands[1].Backfill!.FromCursor.Should().Be("cursor-a"); + + var manifest = new OrchestratorRunManifest( + runId, + "nvd", + "tenant-a", + new OrchestratorBackfillRange("cursor-a", "cursor-z"), + ["sha256:1", "sha256:2"], + "dsse:abc", + DateTimeOffset.Parse("2026-04-19T13:20:00Z")); + await _store.StoreManifestAsync(manifest, TestContext.Current.CancellationToken); + + await using var restartedDataSource = CreateDataSource(); + var restartedStore = new StellaOps.Concelier.Persistence.Postgres.Repositories.PostgresOrchestratorRegistryStore( + restartedDataSource, + TimeProvider.System, + NullLogger.Instance); + var persistedManifest = await restartedStore.GetManifestAsync("tenant-a", "nvd", runId, TestContext.Current.CancellationToken); + + persistedManifest.Should().NotBeNull(); + persistedManifest!.ArtifactHashes.Should().BeEquivalentTo(["sha256:1", "sha256:2"]); + persistedManifest.CursorRange.ToCursor.Should().Be("cursor-z"); + persistedManifest.DsseEnvelopeHash.Should().Be("dsse:abc"); + } + + private ConcelierDataSource CreateDataSource() + { + var options = _fixture.CreateOptions(); + return new ConcelierDataSource(Options.Create(options), NullLogger.Instance); + } + + private static OrchestratorRegistryRecord CreateRegistryRecord(string tenant, string connectorId, string source = "nvd") + { + return new OrchestratorRegistryRecord( + tenant, + connectorId, + source, + ["observations", "timeline"], + "secret:ref", + new OrchestratorSchedule("0 * * * *", "UTC", 2, 60), + new OrchestratorRatePolicy(120, 20, 45), + ["raw-advisory", "timeline"], + $"concelier:{tenant}:{connectorId}", + new OrchestratorEgressGuard(["mirror.example"], false), + DateTimeOffset.Parse("2026-04-19T13:00:00Z"), + DateTimeOffset.Parse("2026-04-19T13:05:00Z")); + } +} diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/TASKS.md b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/TASKS.md index 184c806ed..d7f7539b6 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/TASKS.md +++ b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/TASKS.md @@ -1,7 +1,7 @@ # Concelier Persistence Tests Task Board This board mirrors active sprint tasks for this module. -Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229_049_BE_csproj_audit_maint_tests.md`. +Source of truth: `docs/implplan/SPRINT_20260421_003_Concelier_advisory_connector_runtime_alignment.md` (current); `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229_049_BE_csproj_audit_maint_tests.md` (historical). | Task ID | Status | Notes | | --- | --- | --- | @@ -13,4 +13,8 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | TASK-015-013 | DONE | Added SbomRepository integration coverage for model cards and policy fields. | | TASK-014-003 | DONE | 2026-03-09: added startup-migration registration coverage so Concelier canonical tables bootstrap on fresh deploys and verified `/api/v1/canonical` live after redeploy. | | REALPLAN-007-C | DONE | 2026-04-15: Added `PostgresLeaseStoreTests` and DI registration coverage for durable lease coordination in `vuln.job_leases`. | -| REALPLAN-007-F | DOING | 2026-04-19: Adding persistence coverage for durable advisory observations, affected-symbol storage, and ingest-time symbol extraction against PostgreSQL. | +| REALPLAN-007-F | DONE | 2026-04-19: Added persistence coverage for durable advisory observations, affected-symbol storage, and ingest-time symbol extraction against PostgreSQL; verified with the targeted xUnit helper. | +| REALPLAN-007-G | DONE | 2026-04-19: Added persistence coverage for durable PostgreSQL-backed job runs and internal orchestrator registry state, including restart-safe repository behavior and DI registration checks (`Total: 10, Failed: 0`). | +| CONCELIER-READMODEL-001 | DONE | 2026-04-21: Replaced the advisory-source signature projection with distinct `source_advisory_id` rollups from `vuln.advisory_source_edge`, added PostgreSQL regression coverage, rebuilt Concelier, and verified the live API now reports nonzero advisory totals. | +| CONN-ALIGN-003 | DOING | 2026-04-21: Adding PostgreSQL regression coverage so `/jobs/active` and advisory-source sync backpressure only count lease-backed active runs instead of stale persisted rows. | +| CONN-ALIGN-004-P | DONE | 2026-04-22: Added `PostgresAdvisoryStoreTests` regression coverage so legacy `IAdvisoryStore.UpsertAsync` resolves `source_id` from provenance and keeps KEV source binding recoverable; targeted helper run passed 1/1. | diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/AdvisorySourceEndpointsTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/AdvisorySourceEndpointsTests.cs index 6f969b503..614863d77 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/AdvisorySourceEndpointsTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/AdvisorySourceEndpointsTests.cs @@ -87,6 +87,7 @@ public sealed class AdvisorySourceWebAppFactory : WebApplicationFactory builder.ConfigureServices(services => { + StellaOps.Concelier.Storage.LegacyServiceCollectionExtensions.AddInMemoryStorage(services); services.RemoveAll(); services.AddAuthentication(options => @@ -383,6 +384,10 @@ public sealed class AdvisorySourceWebAppFactory : WebApplicationFactory FreshnessAgeSeconds: 3600, FreshnessStatus: "healthy", SignatureStatus: "signed", + SourceDocumentCount: 220, + CanonicalAdvisoryCount: 220, + CveCount: 220, + VexDocumentCount: 0, TotalAdvisories: 220, SignedAdvisories: 215, UnsignedAdvisories: 5, @@ -405,6 +410,10 @@ public sealed class AdvisorySourceWebAppFactory : WebApplicationFactory FreshnessAgeSeconds: 43200, FreshnessStatus: "stale", SignatureStatus: "unsigned", + SourceDocumentCount: 200, + CanonicalAdvisoryCount: 200, + CveCount: 180, + VexDocumentCount: 0, TotalAdvisories: 200, SignedAdvisories: 0, UnsignedAdvisories: 200, @@ -693,9 +702,350 @@ public sealed class AdvisorySourceEndpointsTests : IClassFixture source.SourceId == "nvd")); Assert.True(nvd.Enabled); + Assert.True(nvd.SyncSupported); + Assert.Equal("source:nvd:fetch", nvd.FetchJobKind); var osv = Assert.Single(payload.Sources.Where(source => source.SourceId == "osv")); Assert.False(osv.Enabled); + Assert.True(osv.SyncSupported); + + var adobe = Assert.Single(payload.Sources.Where(source => source.SourceId == "adobe")); + Assert.True(adobe.SyncSupported); + Assert.Equal("source:adobe:fetch", adobe.FetchJobKind); + + var chromium = Assert.Single(payload.Sources.Where(source => source.SourceId == "chromium")); + Assert.True(chromium.SyncSupported); + Assert.Equal("source:chromium:fetch", chromium.FetchJobKind); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task CatalogEndpoint_MarksCatalogOnlySourceAsUnsupported() + { + using var client = CreateTenantClient(); + + var response = await client.GetAsync("/api/v1/advisory-sources/catalog", CancellationToken.None); + response.EnsureSuccessStatusCode(); + + var payload = await response.Content.ReadFromJsonAsync(cancellationToken: CancellationToken.None); + Assert.NotNull(payload); + + var nvd = Assert.Single(payload!.Items.Where(item => item.Id == "nvd")); + Assert.True(nvd.SyncSupported); + + var cccs = Assert.Single(payload.Items.Where(item => item.Id == "cccs")); + Assert.True(cccs.SyncSupported); + Assert.Equal("source:cccs:fetch", cccs.FetchJobKind); + + var certCc = Assert.Single(payload.Items.Where(item => item.Id == "cert-cc")); + Assert.True(certCc.SyncSupported); + Assert.Equal("source:cert-cc:fetch", certCc.FetchJobKind); + + var krCert = Assert.Single(payload.Items.Where(item => item.Id == "krcert")); + Assert.True(krCert.SyncSupported); + Assert.Equal("source:krcert:fetch", krCert.FetchJobKind); + + var microsoft = Assert.Single(payload.Items.Where(item => item.Id == "microsoft")); + Assert.True(microsoft.SupportsConfiguration); + Assert.True(microsoft.SyncSupported); + Assert.Equal("source:microsoft:fetch", microsoft.FetchJobKind); + + var ghsa = Assert.Single(payload.Items.Where(item => item.Id == "ghsa")); + Assert.True(ghsa.SupportsConfiguration); + Assert.True(ghsa.SyncSupported); + + var cisco = Assert.Single(payload.Items.Where(item => item.Id == "cisco")); + Assert.True(cisco.SupportsConfiguration); + Assert.True(cisco.SyncSupported); + + var oracle = Assert.Single(payload.Items.Where(item => item.Id == "oracle")); + Assert.True(oracle.SupportsConfiguration); + Assert.True(oracle.SyncSupported); + + var adobe = Assert.Single(payload.Items.Where(item => item.Id == "adobe")); + Assert.True(adobe.SupportsConfiguration); + Assert.True(adobe.SyncSupported); + Assert.Equal("source:adobe:fetch", adobe.FetchJobKind); + + var chromium = Assert.Single(payload.Items.Where(item => item.Id == "chromium")); + Assert.True(chromium.SupportsConfiguration); + Assert.True(chromium.SyncSupported); + Assert.Equal("source:chromium:fetch", chromium.FetchJobKind); + + var npm = Assert.Single(payload.Items.Where(item => item.Id == "npm")); + Assert.False(npm.SupportsConfiguration); + Assert.False(npm.SyncSupported); + Assert.Equal("source:npm:fetch", npm.FetchJobKind); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task ConfigurationEndpoint_GetMasksPersistedSensitiveValues() + { + _factory.ResetState(); + await _factory.Services.GetRequiredService().UpsertAsync(new SourceEntity + { + Id = Guid.Parse("dc352b9e-e387-4432-bd79-fc1f893464bf"), + Key = "ghsa", + Name = "GitHub Security Advisories", + SourceType = "ghsa", + Url = "https://github.com/advisories", + Priority = 80, + Enabled = false, + Config = """{"apiToken":"ghp_test_token","mode":"manual"}""", + Metadata = "{}", + CreatedAt = DateTimeOffset.Parse("2026-02-19T00:00:00Z"), + UpdatedAt = DateTimeOffset.Parse("2026-02-19T00:00:00Z") + }, CancellationToken.None); + + using var client = CreateTenantClient(); + + var response = await client.GetAsync("/api/v1/advisory-sources/ghsa/configuration", CancellationToken.None); + response.EnsureSuccessStatusCode(); + + var payload = await response.Content.ReadFromJsonAsync(cancellationToken: CancellationToken.None); + Assert.NotNull(payload); + Assert.Equal("ghsa", payload!.SourceId); + + var apiToken = Assert.Single(payload.Fields.Where(field => field.Key == "apiToken")); + Assert.True(apiToken.Sensitive); + Assert.True(apiToken.Required); + Assert.True(apiToken.HasValue); + Assert.True(apiToken.IsSecretRetained); + Assert.Null(apiToken.Value); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task ConfigurationEndpoint_PutPersistsSettingsAndAllowsEnable() + { + _factory.ResetState(); + using var client = CreateTenantClient(); + + var updateResponse = await client.PutAsJsonAsync( + "/api/v1/advisory-sources/ghsa/configuration", + new SourceConfigurationUpdateRequest + { + Values = new Dictionary + { + ["apiToken"] = "ghp_test_token", + }, + }, + CancellationToken.None); + updateResponse.EnsureSuccessStatusCode(); + + var updatePayload = await updateResponse.Content.ReadFromJsonAsync(cancellationToken: CancellationToken.None); + Assert.NotNull(updatePayload); + var apiToken = Assert.Single(updatePayload!.Fields.Where(field => field.Key == "apiToken")); + Assert.True(apiToken.HasValue); + Assert.True(apiToken.IsSecretRetained); + Assert.Null(apiToken.Value); + + var persisted = _factory.GetSource("ghsa"); + Assert.NotNull(persisted); + Assert.Contains("\"apiToken\":\"ghp_test_token\"", persisted!.Config, StringComparison.Ordinal); + + var enableResponse = await client.PostAsync("/api/v1/advisory-sources/ghsa/enable", content: null, CancellationToken.None); + enableResponse.EnsureSuccessStatusCode(); + + persisted = _factory.GetSource("ghsa"); + Assert.NotNull(persisted); + Assert.True(persisted!.Enabled); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task ConfigurationEndpoint_PutPersistsPublicSourceOverrides() + { + _factory.ResetState(); + using var client = CreateTenantClient(); + + var adobeResponse = await client.PutAsJsonAsync( + "/api/v1/advisory-sources/adobe/configuration", + new SourceConfigurationUpdateRequest + { + Values = new Dictionary + { + ["indexUri"] = "https://mirror.example.internal/adobe/security-bulletin.html", + ["additionalIndexUris"] = "https://mirror.example.internal/adobe/archive-1.html,https://mirror.example.internal/adobe/archive-2.html", + }, + }, + CancellationToken.None); + adobeResponse.EnsureSuccessStatusCode(); + + var chromiumResponse = await client.PutAsJsonAsync( + "/api/v1/advisory-sources/chromium/configuration", + new SourceConfigurationUpdateRequest + { + Values = new Dictionary + { + ["feedUri"] = "https://mirror.example.internal/chromium/atom.xml", + }, + }, + CancellationToken.None); + chromiumResponse.EnsureSuccessStatusCode(); + + var adobe = _factory.GetSource("adobe"); + Assert.NotNull(adobe); + using (var adobeConfig = JsonDocument.Parse(adobe!.Config)) + { + Assert.Equal( + "https://mirror.example.internal/adobe/security-bulletin.html", + adobeConfig.RootElement.GetProperty("indexUri").GetString()); + + var additionalIndexUris = adobeConfig.RootElement.GetProperty("additionalIndexUris").GetString(); + Assert.NotNull(additionalIndexUris); + Assert.Equal( + [ + "https://mirror.example.internal/adobe/archive-1.html", + "https://mirror.example.internal/adobe/archive-2.html", + ], + additionalIndexUris! + .Split(["\r\n", "\n"], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)); + } + + var chromium = _factory.GetSource("chromium"); + Assert.NotNull(chromium); + Assert.Contains("\"feedUri\":\"https://mirror.example.internal/chromium/atom.xml\"", chromium!.Config, StringComparison.Ordinal); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task CheckEndpoint_ReturnsUnsupportedForCatalogOnlySource() + { + using var client = CreateTenantClient(); + + var response = await client.PostAsync("/api/v1/advisory-sources/npm/check", content: null, CancellationToken.None); + response.EnsureSuccessStatusCode(); + + using var payload = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync(CancellationToken.None), cancellationToken: CancellationToken.None); + Assert.Equal("SOURCE_UNSUPPORTED", payload.RootElement.GetProperty("errorCode").GetString()); + Assert.Equal("failed", payload.RootElement.GetProperty("status").GetString()); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task CheckEndpoint_ReturnsConfigRequiredForUnconfiguredGhsa() + { + using var client = CreateTenantClient(); + + var response = await client.PostAsync("/api/v1/advisory-sources/ghsa/check", content: null, CancellationToken.None); + response.EnsureSuccessStatusCode(); + + using var payload = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync(CancellationToken.None), cancellationToken: CancellationToken.None); + Assert.Equal("SOURCE_CONFIG_REQUIRED", payload.RootElement.GetProperty("errorCode").GetString()); + Assert.Equal("failed", payload.RootElement.GetProperty("status").GetString()); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task EnableEndpoint_RejectsCatalogOnlySource() + { + using var client = CreateTenantClient(); + + var response = await client.PostAsync("/api/v1/advisory-sources/npm/enable", content: null, CancellationToken.None); + + Assert.Equal(HttpStatusCode.NotImplemented, response.StatusCode); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task EnableEndpoint_RejectsUnconfiguredGhsa() + { + using var client = CreateTenantClient(); + + var response = await client.PostAsync("/api/v1/advisory-sources/ghsa/enable", content: null, CancellationToken.None); + + Assert.Equal(HttpStatusCode.UnprocessableEntity, response.StatusCode); + using var payload = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync(CancellationToken.None), cancellationToken: CancellationToken.None); + Assert.Equal("source_config_required", payload.RootElement.GetProperty("error").GetString()); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task SyncAllEndpoint_TriggersOnlyRunnableEnabledSources() + { + _factory.ResetState(); + _factory.ResetTriggeredJobs(); + await _factory.Services.GetRequiredService().UpsertAsync(new SourceEntity + { + Id = Guid.Parse("ef8830c6-5d08-4e30-b0ad-ec8d171fcfb2"), + Key = "npm", + Name = "npm advisories", + SourceType = "npm", + Url = "https://example.invalid/npm", + Priority = 50, + Enabled = true, + Config = "{}", + Metadata = "{}", + CreatedAt = DateTimeOffset.Parse("2026-02-19T00:00:00Z"), + UpdatedAt = DateTimeOffset.Parse("2026-02-19T00:00:00Z") + }, CancellationToken.None); + + using var client = CreateTenantClient(); + + var response = await client.PostAsync("/api/v1/advisory-sources/sync", content: null, CancellationToken.None); + response.EnsureSuccessStatusCode(); + + using var document = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync(CancellationToken.None), cancellationToken: CancellationToken.None); + Assert.Equal(2, document.RootElement.GetProperty("totalSources").GetInt32()); + Assert.Equal(1, document.RootElement.GetProperty("totalTriggered").GetInt32()); + + var results = document.RootElement.GetProperty("results").EnumerateArray().ToList(); + var nvd = Assert.Single(results.Where(result => result.GetProperty("sourceId").GetString() == "nvd")); + Assert.Equal("accepted", nvd.GetProperty("outcome").GetString()); + + var npm = Assert.Single(results.Where(result => result.GetProperty("sourceId").GetString() == "npm")); + Assert.Equal("unsupported", npm.GetProperty("outcome").GetString()); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task SyncEndpoint_RejectsUnconfiguredGhsa() + { + using var client = CreateTenantClient(); + + var response = await client.PostAsync("/api/v1/advisory-sources/ghsa/sync", content: null, CancellationToken.None); + + Assert.Equal(HttpStatusCode.UnprocessableEntity, response.StatusCode); + using var payload = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync(CancellationToken.None), cancellationToken: CancellationToken.None); + Assert.Equal("source_config_required", payload.RootElement.GetProperty("error").GetString()); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task SyncAllEndpoint_ReturnsConfigRequiredForEnabledUnconfiguredSource() + { + _factory.ResetState(); + _factory.ResetTriggeredJobs(); + await _factory.Services.GetRequiredService().UpsertAsync(new SourceEntity + { + Id = Guid.Parse("d9543177-ff61-4608-95ec-bb0c4d65af90"), + Key = "ghsa", + Name = "GHSA", + SourceType = "ghsa", + Url = "https://github.com/advisories", + Priority = 80, + Enabled = true, + Config = "{}", + Metadata = "{}", + CreatedAt = DateTimeOffset.Parse("2026-02-19T00:00:00Z"), + UpdatedAt = DateTimeOffset.Parse("2026-02-19T00:00:00Z") + }, CancellationToken.None); + + using var client = CreateTenantClient(); + + var response = await client.PostAsync("/api/v1/advisory-sources/sync", content: null, CancellationToken.None); + response.EnsureSuccessStatusCode(); + + using var document = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync(CancellationToken.None), cancellationToken: CancellationToken.None); + Assert.Equal(2, document.RootElement.GetProperty("totalSources").GetInt32()); + Assert.Equal(1, document.RootElement.GetProperty("totalTriggered").GetInt32()); + + var results = document.RootElement.GetProperty("results").EnumerateArray().ToList(); + var ghsa = Assert.Single(results.Where(result => result.GetProperty("sourceId").GetString() == "ghsa")); + Assert.Equal("config_required", ghsa.GetProperty("outcome").GetString()); + Assert.Equal("SOURCE_CONFIG_REQUIRED", ghsa.GetProperty("errorCode").GetString()); } [Trait("Category", TestCategories.Unit)] @@ -838,8 +1188,7 @@ public sealed class AdvisorySourceEndpointsTests : IClassFixture>(cancellationToken: CancellationToken.None); Assert.NotNull(payload); - var mirror = Assert.Single(payload!); - Assert.Equal("mirror-nvd-001", mirror.MirrorId); + var mirror = Assert.Single(payload!.Where(item => item.MirrorId == "mirror-nvd-001")); Assert.Equal("nvd", mirror.FeedType); Assert.Equal("snap-nvd-20260219", mirror.LatestSnapshotId); Assert.Equal(1, mirror.SnapshotCount); diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/ConcelierHealthEndpointTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/ConcelierHealthEndpointTests.cs index 286c8f688..21bcf973a 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/ConcelierHealthEndpointTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/ConcelierHealthEndpointTests.cs @@ -53,6 +53,7 @@ public sealed class HealthWebAppFactory : WebApplicationFactory builder.ConfigureServices(services => { + StellaOps.Concelier.Storage.LegacyServiceCollectionExtensions.AddInMemoryStorage(services); services.RemoveAll(); services.AddSingleton(); services.AddSingleton(new ConcelierOptions diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/DurableAuditEmissionEndpointTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/DurableAuditEmissionEndpointTests.cs new file mode 100644 index 000000000..df3234602 --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/DurableAuditEmissionEndpointTests.cs @@ -0,0 +1,399 @@ +using System.Net; +using System.Net.Http.Json; +using System.Security.Claims; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Serialization; +using FluentAssertions; +using Microsoft.AspNetCore.Authentication; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Audit.Emission; +using StellaOps.Auth.ServerIntegration.Tenancy; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Persistence.Postgres; +using StellaOps.Concelier.WebService; +using StellaOps.Concelier.WebService.Extensions; +using StellaOps.Concelier.WebService.Jobs; +using StellaOps.Infrastructure.Postgres.Migrations; +using StellaOps.Infrastructure.Postgres.Options; +using StellaOps.Infrastructure.Postgres.Testing; +using StellaOps.TestKit; +using StellaOps.Timeline.Core.Postgres; +using StellaOps.Timeline.WebService.Audit; +using StellaOps.Timeline.WebService.Endpoints; +using StellaOps.TimelineIndexer.Infrastructure; +using Xunit; + +namespace StellaOps.Concelier.WebService.Tests; + +[Collection(DurableRuntimePostgresCollection.Name)] +public sealed class DurableAuditEmissionEndpointTests : IAsyncLifetime +{ + private static readonly JsonSerializerOptions SerializerOptions = CreateJsonOptions(); + private readonly DurableRuntimePostgresFixture _fixture; + + public DurableAuditEmissionEndpointTests(DurableRuntimePostgresFixture fixture) + { + _fixture = fixture; + } + + public ValueTask InitializeAsync() => new(_fixture.TruncateAllTablesAsync()); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + [Trait("Category", TestCategories.Integration)] + [Fact] + public async Task TriggerJob_EmitsAuditEventToTimelineAndPersistsAcrossTimelineRestart() + { + const string tenant = "tenant-a"; + var gate = new BlockingJobGate(); + UnifiedAuditEvent emittedEvent; + + await using (var timelineHost = await TimelineAuditHost.StartAsync(_fixture.ConnectionString)) + using (var timelineClient = timelineHost.CreateClient(tenant)) + using (var concelierFactory = new DurableAuditWebApplicationFactory(_fixture.ConnectionString, timelineHost, gate)) + using (var concelierClient = concelierFactory.CreateClient()) + { + var triggerResponse = await concelierClient.PostAsJsonAsync( + $"/jobs/{DurableAuditWebApplicationFactory.JobKind}", + new + { + trigger = "api", + parameters = new Dictionary(StringComparer.Ordinal) + { + ["tenant"] = tenant, + ["depth"] = 2, + ["mode"] = "full" + } + }); + + triggerResponse.StatusCode.Should().Be(HttpStatusCode.Accepted); + await gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(10)); + gate.Release.TrySetResult(true); + await gate.Completed.Task.WaitAsync(TimeSpan.FromSeconds(10)); + + emittedEvent = await WaitForAuditEventAsync( + timelineClient, + tenant, + DurableAuditWebApplicationFactory.JobKind); + + emittedEvent.Module.Should().Be("concelier"); + emittedEvent.Action.Should().Be("update"); + emittedEvent.Resource.Type.Should().Be("job"); + emittedEvent.Resource.Id.Should().Be(DurableAuditWebApplicationFactory.JobKind); + emittedEvent.Description.Should().Contain(DurableAuditWebApplicationFactory.JobKind); + emittedEvent.TenantId.Should().Be(tenant); + } + + await using var restartedTimelineHost = await TimelineAuditHost.StartAsync(_fixture.ConnectionString); + using var restartedTimelineClient = restartedTimelineHost.CreateClient(tenant); + + var persistedEvent = await WaitForAuditEventAsync( + restartedTimelineClient, + tenant, + DurableAuditWebApplicationFactory.JobKind); + + persistedEvent.Id.Should().Be(emittedEvent.Id); + + using var verifyResponse = await restartedTimelineClient.GetAsync($"/api/v1/audit/chain/verify?tenantId={tenant}"); + verifyResponse.StatusCode.Should().Be(HttpStatusCode.OK); + + using var verifyDocument = JsonDocument.Parse(await verifyResponse.Content.ReadAsStringAsync()); + verifyDocument.RootElement.GetProperty("verified").GetBoolean().Should().BeTrue(); + } + + private static async Task WaitForAuditEventAsync( + HttpClient timelineClient, + string tenant, + string resourceId) + { + for (var attempt = 0; attempt < 40; attempt++) + { + var response = await timelineClient.GetFromJsonAsync( + $"/api/v1/audit/events?tenantId={tenant}&resourceType=job&resourceId={Uri.EscapeDataString(resourceId)}&limit=20", + SerializerOptions); + + var auditEvent = response?.Items.SingleOrDefault(item => + string.Equals(item.Module, "concelier", StringComparison.OrdinalIgnoreCase) && + string.Equals(item.Resource.Type, "job", StringComparison.OrdinalIgnoreCase) && + string.Equals(item.Resource.Id, resourceId, StringComparison.Ordinal)); + + if (auditEvent is not null) + { + return auditEvent; + } + + await Task.Delay(100); + } + + throw new Xunit.Sdk.XunitException($"Timed out waiting for audit event for resource '{resourceId}'."); + } + + private static JsonSerializerOptions CreateJsonOptions() + { + var options = new JsonSerializerOptions(JsonSerializerDefaults.Web); + options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase)); + return options; + } + + private sealed class BlockingJobGate + { + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Completed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + private sealed class TimelineAuditHost : IAsyncDisposable + { + private readonly WebApplication _app; + private readonly TestServer _server; + + private TimelineAuditHost(WebApplication app, TestServer server) + { + _app = app; + _server = server; + } + + public static async Task StartAsync(string connectionString) + { + var builder = WebApplication.CreateBuilder(new WebApplicationOptions + { + EnvironmentName = "Testing" + }); + + builder.WebHost.UseTestServer(); + builder.Services.Configure(options => + { + options.ConnectionString = connectionString; + }); + builder.Services.Configure(options => + { + options.AuthorityBaseUrl = string.Empty; + options.JobEngineBaseUrl = string.Empty; + options.PolicyBaseUrl = string.Empty; + options.EvidenceLockerBaseUrl = string.Empty; + options.NotifyBaseUrl = string.Empty; + options.RequestTimeoutSeconds = 1; + }); + builder.Services.AddHttpClient(HttpUnifiedAuditEventProvider.ClientName, (provider, client) => + { + var options = provider.GetRequiredService>().Value; + client.Timeout = TimeSpan.FromSeconds(Math.Max(1, options.RequestTimeoutSeconds)); + }); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddStartupMigrations( + schemaName: TimelineIndexerDataSource.DefaultSchemaName, + moduleName: "TimelineIndexer", + migrationsAssembly: typeof(TimelineIndexerDataSource).Assembly, + connectionStringSelector: options => options.ConnectionString); + builder.Services.AddStartupMigrations( + schemaName: TimelineCoreDataSource.DefaultSchemaName, + moduleName: "TimelineAudit", + migrationsAssembly: typeof(TimelineCoreDataSource).Assembly, + connectionStringSelector: options => options.ConnectionString); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddStellaOpsTenantServices(); + builder.Services.AddAuthorization(options => + { + options.AddPolicy("Timeline.Read", policy => policy.RequireAssertion(_ => true)); + options.AddPolicy("Timeline.Write", policy => policy.RequireAssertion(_ => true)); + options.AddPolicy("Timeline.Admin", policy => policy.RequireAssertion(_ => true)); + }); + builder.Services.AddAuthentication(options => + { + options.DefaultAuthenticateScheme = TimelineTestAuthHandler.SchemeName; + options.DefaultChallengeScheme = TimelineTestAuthHandler.SchemeName; + }) + .AddScheme( + TimelineTestAuthHandler.SchemeName, + _ => { }); + + var app = builder.Build(); + app.UseAuthentication(); + app.UseAuthorization(); + app.UseStellaOpsTenantMiddleware(); + app.MapUnifiedAuditEndpoints(); + await app.StartAsync(); + + return new TimelineAuditHost(app, app.GetTestServer()); + } + + public HttpClient CreateClient(string tenant) + { + var client = _app.GetTestClient(); + client.DefaultRequestHeaders.Add("x-tenant-id", tenant); + return client; + } + + public HttpMessageHandler CreateHandler() => _server.CreateHandler(); + + public async ValueTask DisposeAsync() + { + await _app.StopAsync(); + await _app.DisposeAsync(); + } + } + + private sealed class DurableAuditWebApplicationFactory : WebApplicationFactory + { + public const string JobKind = "durable-runtime"; + + private readonly BlockingJobGate _gate; + private readonly TimelineAuditHost _timelineHost; + private readonly Dictionary _previousEnvironment = new(StringComparer.OrdinalIgnoreCase); + + public DurableAuditWebApplicationFactory( + string connectionString, + TimelineAuditHost timelineHost, + BlockingJobGate gate) + { + _timelineHost = timelineHost; + _gate = gate; + + SetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__CONNECTIONSTRING", connectionString); + SetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__ENABLED", "true"); + SetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__COMMANDTIMEOUTSECONDS", "30"); + SetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__SCHEMANAME", "vuln"); + SetEnvironmentVariable("CONCELIER_POSTGRES_DSN", connectionString); + SetEnvironmentVariable("CONCELIER_SKIP_OPTIONS_VALIDATION", "1"); + SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLED", "false"); + SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLELOGGING", "false"); + SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLETRACING", "false"); + SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLEMETRICS", "false"); + SetEnvironmentVariable("CONCELIER_AUTHORITY__ENABLED", "false"); + SetEnvironmentVariable("CONCELIER_AUTHORITY__REQUIREDSCOPES__0", "concelier.jobs.trigger"); + SetEnvironmentVariable("STELLAOPS_TIMELINE_URL", "http://timeline.test"); + SetEnvironmentVariable("DOTNET_ENVIRONMENT", "Production"); + SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Production"); + } + + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.UseEnvironment("Production"); + builder.ConfigureServices(services => + { + services.AddHttpContextAccessor(); + services.AddAuthentication("Test") + .AddScheme("Test", _ => { }) + .AddScheme("StellaOpsBearer", _ => { }); + services.AddAuthorization(options => + { + options.AddPolicy("Concelier.Sources.Manage", policy => policy.RequireAssertion(_ => true)); + options.AddPolicy("Concelier.Advisories.Read", policy => policy.RequireAssertion(_ => true)); + }); + + services.AddHttpClient(HttpAuditEventEmitter.HttpClientName) + .ConfigurePrimaryHttpMessageHandler(() => _timelineHost.CreateHandler()); + + services.TryAddSingleton>(_ => Guid.NewGuid); + services.RemoveAll(); + + services.AddSingleton(_gate); + services.AddTransient(); + services.PostConfigure(options => + { + options.Definitions[JobKind] = new JobDefinition( + JobKind, + typeof(DurableBlockingJob), + TimeSpan.FromSeconds(30), + TimeSpan.FromSeconds(5), + null, + true); + }); + }); + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + foreach (var entry in _previousEnvironment) + { + Environment.SetEnvironmentVariable(entry.Key, entry.Value); + } + } + + base.Dispose(disposing); + } + + private void SetEnvironmentVariable(string name, string? value) + { + _previousEnvironment[name] = Environment.GetEnvironmentVariable(name); + Environment.SetEnvironmentVariable(name, value); + } + } + + private sealed class DurableBlockingJob(BlockingJobGate gate) : IJob + { + public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + { + gate.Started.TrySetResult(true); + await gate.Release.Task.WaitAsync(cancellationToken); + gate.Completed.TrySetResult(true); + } + } + + private sealed class AlwaysAllowAuthHandler : AuthenticationHandler + { + public AlwaysAllowAuthHandler( + IOptionsMonitor options, + ILoggerFactory logger, + UrlEncoder encoder) + : base(options, logger, encoder) + { + } + + protected override Task HandleAuthenticateAsync() + { + var identity = new ClaimsIdentity( + [ + new Claim(ClaimTypes.NameIdentifier, "test-user"), + new Claim(ClaimTypes.Name, "Test User"), + new Claim("scope", "concelier.jobs.trigger"), + new Claim("stellaops:tenant", "tenant-a") + ], + Scheme.Name); + var principal = new ClaimsPrincipal(identity); + var ticket = new AuthenticationTicket(principal, Scheme.Name); + return Task.FromResult(AuthenticateResult.Success(ticket)); + } + } + + private sealed class TimelineTestAuthHandler : AuthenticationHandler + { + public const string SchemeName = "TimelineTest"; + + public TimelineTestAuthHandler( + IOptionsMonitor options, + ILoggerFactory logger, + UrlEncoder encoder) + : base(options, logger, encoder) + { + } + + protected override Task HandleAuthenticateAsync() + { + var identity = new ClaimsIdentity( + [ + new Claim(ClaimTypes.NameIdentifier, "timeline-user"), + new Claim("scope", "timeline:read timeline:write timeline:admin"), + new Claim("stellaops:tenant", "tenant-a") + ], + Scheme.Name); + var principal = new ClaimsPrincipal(identity); + var ticket = new AuthenticationTicket(principal, Scheme.Name); + return Task.FromResult(AuthenticateResult.Success(ticket)); + } + } +} diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/DurableRuntimeEndpointTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/DurableRuntimeEndpointTests.cs new file mode 100644 index 000000000..a7ebfb234 --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/DurableRuntimeEndpointTests.cs @@ -0,0 +1,425 @@ +using System.Net; +using System.Net.Http.Json; +using System.Reflection; +using System.Security.Claims; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Serialization; +using FluentAssertions; +using Microsoft.AspNetCore.Authentication; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; +using StellaOps.Concelier.Core.Federation; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Core.Observations; +using StellaOps.Concelier.Core.Orchestration; +using StellaOps.Concelier.Core.Raw; +using StellaOps.Concelier.Persistence.Postgres; +using StellaOps.Concelier.Storage; +using StellaOps.Concelier.WebService; +using StellaOps.Concelier.WebService.Extensions; +using StellaOps.Concelier.WebService.Jobs; +using StellaOps.Concelier.WebService.Services; +using StellaOps.Infrastructure.Postgres.Testing; +using StellaOps.Replay.Core.FeedSnapshot; +using StellaOps.TestKit; +using Xunit; + +namespace StellaOps.Concelier.WebService.Tests; + +[CollectionDefinition(Name)] +public sealed class DurableRuntimePostgresCollection : ICollectionFixture +{ + public const string Name = "Concelier Durable Runtime Postgres"; +} + +public sealed class DurableRuntimePostgresFixture : PostgresIntegrationFixture +{ + protected override Assembly? GetMigrationAssembly() + => typeof(ConcelierDataSource).Assembly; + + protected override string GetModuleName() => "Concelier"; +} + +[Collection(DurableRuntimePostgresCollection.Name)] +public sealed class DurableRuntimeEndpointTests : IAsyncLifetime +{ + private static readonly JsonSerializerOptions SerializerOptions = CreateJsonOptions(); + private readonly DurableRuntimePostgresFixture _fixture; + + public DurableRuntimeEndpointTests(DurableRuntimePostgresFixture fixture) + { + _fixture = fixture; + } + + public ValueTask InitializeAsync() => new(_fixture.TruncateAllTablesAsync()); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task JobsEndpoints_PersistRuntimeStateAcrossHostRestart() + { + var gate = new BlockingJobGate(); + Guid runId; + + using (var factory = new DurableRuntimeWebApplicationFactory(_fixture.ConnectionString, gate)) + using (var client = factory.CreateClient()) + { + var triggerResponse = await client.PostAsJsonAsync( + $"/jobs/{DurableRuntimeWebApplicationFactory.JobKind}", + new + { + trigger = "api", + parameters = new Dictionary(StringComparer.Ordinal) + { + ["tenant"] = "tenant-a", + ["depth"] = 2, + ["mode"] = "full" + } + }); + + triggerResponse.StatusCode.Should().Be(HttpStatusCode.Accepted); + triggerResponse.Headers.Location.Should().NotBeNull(); + + var accepted = await triggerResponse.Content.ReadFromJsonAsync(SerializerOptions); + accepted.Should().NotBeNull(); + accepted!.Kind.Should().Be(DurableRuntimeWebApplicationFactory.JobKind); + accepted.Status.Should().Be(JobRunStatus.Running); + accepted.Trigger.Should().Be("api"); + runId = accepted.RunId; + + await gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(10)); + + var activeRuns = await WaitForAsync( + async () => await client.GetFromJsonAsync>("/jobs/active", SerializerOptions) ?? [], + runs => runs.Any(run => run.RunId == runId && run.Status == JobRunStatus.Running), + "Timed out waiting for the triggered job run to appear in /jobs/active."); + + activeRuns.Should().ContainSingle(run => run.RunId == runId); + + var runningSnapshot = await client.GetFromJsonAsync($"/jobs/{runId}", SerializerOptions); + runningSnapshot.Should().NotBeNull(); + runningSnapshot!.Status.Should().Be(JobRunStatus.Running); + + gate.Release.TrySetResult(true); + await gate.Completed.Task.WaitAsync(TimeSpan.FromSeconds(10)); + + var completed = await WaitForAsync( + async () => await client.GetFromJsonAsync($"/jobs/{runId}", SerializerOptions), + run => run is not null && run.Status == JobRunStatus.Succeeded, + "Timed out waiting for the durable job run to complete."); + + completed!.CompletedAt.Should().NotBeNull(); + } + + using var restartedFactory = new DurableRuntimeWebApplicationFactory(_fixture.ConnectionString, new BlockingJobGate()); + using var restartedClient = restartedFactory.CreateClient(); + + var persistedRun = await restartedClient.GetFromJsonAsync($"/jobs/{runId}", SerializerOptions); + persistedRun.Should().NotBeNull(); + persistedRun!.Status.Should().Be(JobRunStatus.Succeeded); + persistedRun.Parameters.Should().ContainKey("tenant"); + + var definitions = await restartedClient.GetFromJsonAsync>("/jobs/definitions", SerializerOptions); + definitions.Should().NotBeNull(); + var persistedDefinition = definitions!.Single(definition => definition.Kind == DurableRuntimeWebApplicationFactory.JobKind); + persistedDefinition.LastRun.Should().NotBeNull(); + persistedDefinition.LastRun!.RunId.Should().Be(runId); + persistedDefinition.LastRun.Status.Should().Be(JobRunStatus.Succeeded); + + var definition = await restartedClient.GetFromJsonAsync( + $"/jobs/definitions/{DurableRuntimeWebApplicationFactory.JobKind}", + SerializerOptions); + definition.Should().NotBeNull(); + definition!.Kind.Should().Be(DurableRuntimeWebApplicationFactory.JobKind); + definition.LastRun.Should().NotBeNull(); + definition.LastRun!.RunId.Should().Be(runId); + + var runsByDefinition = await restartedClient.GetFromJsonAsync>( + $"/jobs/definitions/{DurableRuntimeWebApplicationFactory.JobKind}/runs", + SerializerOptions); + runsByDefinition.Should().NotBeNull(); + runsByDefinition!.Should().ContainSingle(run => run.RunId == runId && run.Status == JobRunStatus.Succeeded); + + var recentRuns = await restartedClient.GetFromJsonAsync>( + $"/jobs?kind={DurableRuntimeWebApplicationFactory.JobKind}&limit=10", + SerializerOptions); + recentRuns.Should().NotBeNull(); + recentRuns!.Should().ContainSingle(run => run.RunId == runId && run.Status == JobRunStatus.Succeeded); + + var activeAfterRestart = await restartedClient.GetFromJsonAsync>("/jobs/active", SerializerOptions); + activeAfterRestart.Should().NotBeNull(); + activeAfterRestart!.Should().BeEmpty(); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task OrchestratorEndpoints_PersistRegistryHeartbeatAndCommandsAcrossHostRestart() + { + const string tenant = "tenant-a"; + const string connectorId = "demo-connector"; + var runId = Guid.Parse("11111111-1111-1111-1111-111111111111"); + + using (var factory = new DurableRuntimeWebApplicationFactory(_fixture.ConnectionString, new BlockingJobGate())) + using (var client = CreateTenantClient(factory, tenant)) + { + var registryResponse = await client.PostAsJsonAsync("/internal/orch/registry", new + { + connectorId, + source = "nvd", + capabilities = new[] { "ingest", "timeline" }, + authRef = "secret:ref", + schedule = new { cron = "0 * * * *", timeZone = "UTC", maxParallelRuns = 2, maxLagMinutes = 60 }, + ratePolicy = new { rpm = 120, burst = 20, cooldownSeconds = 45 }, + artifactKinds = new[] { "raw-advisory", "timeline" }, + lockKey = $"concelier:{tenant}:{connectorId}", + egressGuard = new { allowlist = new[] { "mirror.example" }, airgapMode = false } + }); + + registryResponse.StatusCode.Should().Be(HttpStatusCode.Accepted); + + var heartbeatResponse = await client.PostAsJsonAsync("/internal/orch/heartbeat", new + { + connectorId, + runId, + sequence = 3, + status = "succeeded", + progress = 100, + queueDepth = 0, + lastArtifactHash = "sha256:artifact", + lastArtifactKind = "normalized", + timestampUtc = DateTimeOffset.Parse("2026-04-19T13:02:00Z") + }); + + heartbeatResponse.StatusCode.Should().Be(HttpStatusCode.Accepted); + + var commandResponse = await client.PostAsJsonAsync("/internal/orch/commands", new + { + connectorId, + runId, + sequence = 2, + command = "backfill", + backfill = new { fromCursor = "cursor-a", toCursor = "cursor-z" }, + expiresAt = DateTimeOffset.Parse("2100-01-01T00:06:00Z") + }); + + commandResponse.StatusCode.Should().Be(HttpStatusCode.Accepted); + + var commandUrl = $"/internal/orch/commands?connectorId={connectorId}&runId={runId}"; + var pendingCommands = await client.GetFromJsonAsync>(commandUrl, SerializerOptions); + pendingCommands.Should().NotBeNull(); + var queuedCommand = pendingCommands!.Single(command => command.Sequence == 2); + queuedCommand.Command.Should().Be(OrchestratorCommandKind.Backfill); + queuedCommand.Backfill.Should().NotBeNull(); + queuedCommand.Backfill!.FromCursor.Should().Be("cursor-a"); + queuedCommand.Backfill.ToCursor.Should().Be("cursor-z"); + } + + using var restartedFactory = new DurableRuntimeWebApplicationFactory(_fixture.ConnectionString, new BlockingJobGate()); + using var restartedClient = CreateTenantClient(restartedFactory, tenant); + + var persistedCommands = await restartedClient.GetFromJsonAsync>( + $"/internal/orch/commands?connectorId={connectorId}&runId={runId}", + SerializerOptions); + persistedCommands.Should().NotBeNull(); + persistedCommands!.Should().ContainSingle(command => command.Sequence == 2 && command.Command == OrchestratorCommandKind.Backfill); + + using var scope = restartedFactory.Services.CreateScope(); + var store = scope.ServiceProvider.GetRequiredService(); + + var registry = await store.GetAsync(tenant, connectorId, TestContext.Current.CancellationToken); + registry.Should().NotBeNull(); + registry!.Source.Should().Be("nvd"); + registry.Schedule.Cron.Should().Be("0 * * * *"); + registry.RatePolicy.Rpm.Should().Be(120); + registry.EgressGuard.Allowlist.Should().BeEquivalentTo(["mirror.example"]); + + var latestHeartbeat = await store.GetLatestHeartbeatAsync(tenant, connectorId, runId, TestContext.Current.CancellationToken); + latestHeartbeat.Should().NotBeNull(); + latestHeartbeat!.Sequence.Should().Be(3); + latestHeartbeat.Status.Should().Be(OrchestratorHeartbeatStatus.Succeeded); + latestHeartbeat.LastArtifactHash.Should().Be("sha256:artifact"); + } + + private static HttpClient CreateTenantClient(WebApplicationFactory factory, string tenant) + { + var client = factory.CreateClient(); + client.DefaultRequestHeaders.Add(Program.TenantHeaderName, tenant); + return client; + } + + private static async Task WaitForAsync( + Func> action, + Func predicate, + string failureMessage) + { + for (var attempt = 0; attempt < 40; attempt++) + { + var result = await action(); + if (predicate(result)) + { + return result; + } + + await Task.Delay(100); + } + + throw new Xunit.Sdk.XunitException(failureMessage); + } + + private static JsonSerializerOptions CreateJsonOptions() + { + var options = new JsonSerializerOptions(JsonSerializerDefaults.Web); + options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase)); + return options; + } + + private sealed class BlockingJobGate + { + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Completed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + private sealed class DurableRuntimeWebApplicationFactory : WebApplicationFactory + { + public const string JobKind = "durable-runtime"; + + private readonly string _connectionString; + private readonly BlockingJobGate _gate; + private readonly Dictionary _previousEnvironment = new(StringComparer.OrdinalIgnoreCase); + + public DurableRuntimeWebApplicationFactory(string connectionString, BlockingJobGate gate) + { + _connectionString = connectionString; + _gate = gate; + + SetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__CONNECTIONSTRING", connectionString); + SetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__ENABLED", "true"); + SetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__COMMANDTIMEOUTSECONDS", "30"); + SetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__SCHEMANAME", "vuln"); + SetEnvironmentVariable("CONCELIER_POSTGRES_DSN", connectionString); + SetEnvironmentVariable("CONCELIER_SKIP_OPTIONS_VALIDATION", "1"); + SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLED", "false"); + SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLELOGGING", "false"); + SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLETRACING", "false"); + SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLEMETRICS", "false"); + SetEnvironmentVariable("CONCELIER_AUTHORITY__ENABLED", "false"); + SetEnvironmentVariable("CONCELIER_AUTHORITY__REQUIREDSCOPES__0", "concelier.jobs.trigger"); + SetEnvironmentVariable("DOTNET_ENVIRONMENT", "Production"); + SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Production"); + } + + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.UseEnvironment("Production"); + builder.ConfigureServices(services => + { + services.AddHttpContextAccessor(); + services.AddAuthentication("Test") + .AddScheme("Test", _ => { }) + .AddScheme("StellaOpsBearer", _ => { }); + services.AddAuthorization(options => + { + options.AddPolicy("Concelier.Sources.Manage", policy => policy.RequireAssertion(_ => true)); + options.AddPolicy("Concelier.Advisories.Read", policy => policy.RequireAssertion(_ => true)); + }); + + services.TryAddSingleton>(_ => Guid.NewGuid); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.TryAddSingleton(Mock.Of()); + services.TryAddSingleton(Mock.Of()); + services.TryAddSingleton(Mock.Of()); + services.TryAddSingleton(CreateFeedSnapshotCoordinator()); + services.RemoveAll(); + services.TryAddSingleton(Mock.Of()); + services.RemoveAll(); + services.TryAddSingleton(Mock.Of()); + services.RemoveAll(); + + services.AddSingleton(_gate); + services.AddTransient(); + services.PostConfigure(options => + { + options.Definitions[JobKind] = new JobDefinition( + JobKind, + typeof(DurableBlockingJob), + TimeSpan.FromSeconds(30), + TimeSpan.FromSeconds(5), + null, + true); + }); + }); + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + foreach (var entry in _previousEnvironment) + { + Environment.SetEnvironmentVariable(entry.Key, entry.Value); + } + } + + base.Dispose(disposing); + } + + private void SetEnvironmentVariable(string name, string? value) + { + _previousEnvironment[name] = Environment.GetEnvironmentVariable(name); + Environment.SetEnvironmentVariable(name, value); + } + + private static IFeedSnapshotCoordinator CreateFeedSnapshotCoordinator() + { + var mock = new Mock(); + mock.SetupGet(x => x.RegisteredSources).Returns(Array.Empty()); + return mock.Object; + } + } + + private sealed class DurableBlockingJob(BlockingJobGate gate) : IJob + { + public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + { + gate.Started.TrySetResult(true); + await gate.Release.Task.WaitAsync(cancellationToken); + gate.Completed.TrySetResult(true); + } + } + + private sealed class AlwaysAllowAuthHandler : AuthenticationHandler + { + public AlwaysAllowAuthHandler( + IOptionsMonitor options, + ILoggerFactory logger, + UrlEncoder encoder) + : base(options, logger, encoder) + { + } + + protected override Task HandleAuthenticateAsync() + { + var identity = new ClaimsIdentity( + [ + new Claim(ClaimTypes.NameIdentifier, "test-user"), + new Claim(ClaimTypes.Name, "Test User"), + new Claim("scope", "concelier.jobs.trigger") + ], + Scheme.Name); + var principal = new ClaimsPrincipal(identity); + var ticket = new AuthenticationTicket(principal, Scheme.Name); + return Task.FromResult(AuthenticateResult.Success(ticket)); + } + } +} diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/FederationEndpointTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/FederationEndpointTests.cs index f857a29c4..b6f941009 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/FederationEndpointTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/FederationEndpointTests.cs @@ -277,6 +277,8 @@ public sealed class FederationEndpointTests builder.ConfigureServices(services => { + StellaOps.Concelier.Storage.LegacyServiceCollectionExtensions.AddInMemoryStorage(services); + // Register test authentication and authorization services.AddAuthentication(options => { diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/Fixtures/ConcelierApplicationFactory.cs b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/Fixtures/ConcelierApplicationFactory.cs index bf453142a..5bb02bbfd 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/Fixtures/ConcelierApplicationFactory.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/Fixtures/ConcelierApplicationFactory.cs @@ -88,6 +88,8 @@ public class ConcelierApplicationFactory : WebApplicationFactory builder.ConfigureServices(services => { + StellaOps.Concelier.Storage.LegacyServiceCollectionExtensions.AddInMemoryStorage(services); + // Register test authentication so endpoints requiring auth don't fail services.AddAuthentication(options => { diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/InterestScoreEndpointTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/InterestScoreEndpointTests.cs index b9cfbbf30..6515c650d 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/InterestScoreEndpointTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/InterestScoreEndpointTests.cs @@ -382,6 +382,8 @@ public sealed class InterestScoreEndpointTests : IClassFixture { + StellaOps.Concelier.Storage.LegacyServiceCollectionExtensions.AddInMemoryStorage(services); + // Register test authentication and authorization services.AddAuthentication(options => { diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/JobRegistrationExtensionsTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/JobRegistrationExtensionsTests.cs index 58e85fdc3..b4bd694f4 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/JobRegistrationExtensionsTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/JobRegistrationExtensionsTests.cs @@ -1,8 +1,16 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.Cccs; +using StellaOps.Concelier.Connector.CertCc; +using StellaOps.Concelier.Connector.Nvd; +using StellaOps.Concelier.Connector.Cve; +using StellaOps.Concelier.Connector.Kisa; +using StellaOps.Concelier.Connector.Vndr.Adobe; using StellaOps.Concelier.Connector.Vndr.Cisco; using StellaOps.Concelier.Connector.Vndr.Cisco.Configuration; +using StellaOps.Concelier.Connector.Vndr.Chromium; +using StellaOps.Concelier.Connector.Vndr.Msrc; using StellaOps.Concelier.Connector.StellaOpsMirror; using StellaOps.Concelier.Connector.StellaOpsMirror.Settings; using StellaOps.Concelier.Core.Jobs; @@ -64,6 +72,118 @@ public sealed class JobRegistrationExtensionsTests Assert.DoesNotContain("source:stellaops-mirror:fetch", schedulerOptions.Definitions.Keys); } + [Fact] + public void AddBuiltInConcelierJobs_RegistersPrimaryCveAndNvdPipelines() + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddOptions(); + + services.AddBuiltInConcelierJobs(new ConfigurationBuilder().Build()); + + using var provider = services.BuildServiceProvider(); + var schedulerOptions = provider.GetRequiredService>().Value; + + Assert.Contains("source:cve:fetch", schedulerOptions.Definitions.Keys); + Assert.Contains("source:cve:parse", schedulerOptions.Definitions.Keys); + Assert.Contains("source:cve:map", schedulerOptions.Definitions.Keys); + Assert.Contains("source:nvd:fetch", schedulerOptions.Definitions.Keys); + Assert.Contains("source:nvd:parse", schedulerOptions.Definitions.Keys); + Assert.Contains("source:nvd:map", schedulerOptions.Definitions.Keys); + Assert.Contains(services, descriptor => descriptor.ServiceType == typeof(CveConnector)); + Assert.Contains(services, descriptor => descriptor.ServiceType == typeof(NvdConnector)); + } + + [Fact] + public void AddBuiltInConcelierJobs_RegistersImplementedRegionalAndVendorPipelines() + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddOptions(); + + services.AddBuiltInConcelierJobs(new ConfigurationBuilder().Build()); + + using var provider = services.BuildServiceProvider(); + var schedulerOptions = provider.GetRequiredService>().Value; + + Assert.Contains("source:cert-cc:fetch", schedulerOptions.Definitions.Keys); + Assert.Contains("source:cert-cc:parse", schedulerOptions.Definitions.Keys); + Assert.Contains("source:cert-cc:map", schedulerOptions.Definitions.Keys); + Assert.Contains("source:cccs:fetch", schedulerOptions.Definitions.Keys); + Assert.Contains("source:cccs:parse", schedulerOptions.Definitions.Keys); + Assert.Contains("source:cccs:map", schedulerOptions.Definitions.Keys); + Assert.Contains("source:krcert:fetch", schedulerOptions.Definitions.Keys); + Assert.Contains("source:krcert:parse", schedulerOptions.Definitions.Keys); + Assert.Contains("source:krcert:map", schedulerOptions.Definitions.Keys); + Assert.Contains("source:microsoft:fetch", schedulerOptions.Definitions.Keys); + Assert.Contains("source:microsoft:parse", schedulerOptions.Definitions.Keys); + Assert.Contains("source:microsoft:map", schedulerOptions.Definitions.Keys); + Assert.Contains("source:us-cert:fetch", schedulerOptions.Definitions.Keys); + Assert.Contains("source:us-cert:parse", schedulerOptions.Definitions.Keys); + Assert.Contains("source:us-cert:map", schedulerOptions.Definitions.Keys); + Assert.Contains("source:kaspersky-ics:fetch", schedulerOptions.Definitions.Keys); + Assert.Contains("source:kaspersky-ics:parse", schedulerOptions.Definitions.Keys); + Assert.Contains("source:kaspersky-ics:map", schedulerOptions.Definitions.Keys); + Assert.Contains("source:fstec-bdu:fetch", schedulerOptions.Definitions.Keys); + Assert.Contains("source:fstec-bdu:parse", schedulerOptions.Definitions.Keys); + Assert.Contains("source:fstec-bdu:map", schedulerOptions.Definitions.Keys); + Assert.Contains("source:nkcki:fetch", schedulerOptions.Definitions.Keys); + Assert.Contains("source:nkcki:parse", schedulerOptions.Definitions.Keys); + Assert.Contains("source:nkcki:map", schedulerOptions.Definitions.Keys); + Assert.Contains("source:apple:fetch", schedulerOptions.Definitions.Keys); + Assert.Contains("source:apple:parse", schedulerOptions.Definitions.Keys); + Assert.Contains("source:apple:map", schedulerOptions.Definitions.Keys); + Assert.Contains("source:adobe:fetch", schedulerOptions.Definitions.Keys); + Assert.Contains("source:adobe:parse", schedulerOptions.Definitions.Keys); + Assert.Contains("source:adobe:map", schedulerOptions.Definitions.Keys); + Assert.Contains("source:chromium:fetch", schedulerOptions.Definitions.Keys); + Assert.Contains("source:chromium:parse", schedulerOptions.Definitions.Keys); + Assert.Contains("source:chromium:map", schedulerOptions.Definitions.Keys); + Assert.Contains("source:cisco:fetch", schedulerOptions.Definitions.Keys); + Assert.Contains("source:cisco:parse", schedulerOptions.Definitions.Keys); + Assert.Contains("source:cisco:map", schedulerOptions.Definitions.Keys); + Assert.Contains("source:oracle:fetch", schedulerOptions.Definitions.Keys); + Assert.Contains("source:oracle:parse", schedulerOptions.Definitions.Keys); + Assert.Contains("source:oracle:map", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:ics-cisa:fetch", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:ics-cisa:parse", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:ics-cisa:map", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:ics-kaspersky:fetch", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:ics-kaspersky:parse", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:ics-kaspersky:map", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:ru-bdu:fetch", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:ru-bdu:parse", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:ru-bdu:map", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:ru-nkcki:fetch", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:ru-nkcki:parse", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:ru-nkcki:map", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:vndr-adobe:fetch", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:vndr-adobe:parse", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:vndr-adobe:map", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:vndr-apple:fetch", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:vndr-apple:parse", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:vndr-apple:map", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:vndr-cisco:fetch", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:vndr-cisco:parse", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:vndr-cisco:map", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:vndr-chromium:fetch", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:vndr-chromium:parse", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:vndr-chromium:map", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:vndr.msrc:fetch", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:vndr.msrc:parse", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:vndr.msrc:map", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:vndr-oracle:fetch", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:vndr-oracle:parse", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:vndr-oracle:map", schedulerOptions.Definitions.Keys); + + Assert.Contains(services, descriptor => descriptor.ServiceType == typeof(CccsConnector)); + Assert.Contains(services, descriptor => descriptor.ServiceType == typeof(CertCcConnector)); + Assert.Contains(services, descriptor => descriptor.ServiceType == typeof(KisaConnector)); + Assert.Contains(services, descriptor => descriptor.ServiceType == typeof(AdobeConnector)); + Assert.Contains(services, descriptor => descriptor.ServiceType == typeof(ChromiumConnector)); + Assert.Contains(services, descriptor => descriptor.ServiceType == typeof(MsrcConnector)); + } + [Fact] public void AddBuiltInConcelierJobs_DoesNotPromoteConnectorOptionValidationToHostStartup() { diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/LeaseStoreWiringTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/LeaseStoreWiringTests.cs index c1d35afac..4c52275fc 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/LeaseStoreWiringTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/LeaseStoreWiringTests.cs @@ -43,6 +43,7 @@ public sealed class LeaseStoreWiringTests builder.UseEnvironment("Testing"); builder.ConfigureServices(services => { + StellaOps.Concelier.Storage.LegacyServiceCollectionExtensions.AddInMemoryStorage(services); services.RemoveAll(); }); } diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/OrchestratorEndpointsTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/OrchestratorEndpointsTests.cs index 288f4a4f5..808f1d4ba 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/OrchestratorEndpointsTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/OrchestratorEndpointsTests.cs @@ -52,6 +52,7 @@ public sealed class OrchestratorTestWebAppFactory : WebApplicationFactory { + StellaOps.Concelier.Storage.LegacyServiceCollectionExtensions.AddInMemoryStorage(services); services.RemoveAll(); services.AddSingleton(); services.RemoveAll(); @@ -158,4 +159,3 @@ public sealed class OrchestratorEndpointsTests : IClassFixture + + + + + + diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/TASKS.md b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/TASKS.md index bb91446db..15da72cfb 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/TASKS.md +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/TASKS.md @@ -1,7 +1,7 @@ # Concelier WebService Tests Task Board This board mirrors active sprint tasks for this module. -Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229_049_BE_csproj_audit_maint_tests.md`. +Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229_049_BE_csproj_audit_maint_tests.md`, `docs/implplan/SPRINT_20260408_004_Timeline_unified_audit_sink.md`. | Task ID | Status | Notes | | --- | --- | --- | @@ -12,7 +12,11 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | NOMOCK-003 | DONE | 2026-04-18: Extended `UnsupportedRuntimeWiringTests` and hardened `WebServiceEndpointsTests` mirror happy-path/auth/rate-limit proof so runtime evidence seeds mirror state through persisted stores instead of config-only domains. | | REALPLAN-007-C | DONE | 2026-04-15: Added `LeaseStoreWiringTests` so the live WebService host resolves `ILeaseStore` to `PostgresLeaseStore`. | | REALPLAN-007-E | DONE | 2026-04-17: Extended `UnsupportedRuntimeWiringTests` to prove affected-symbol services resolve to explicit unsupported runtime implementations and `/v1/signals/symbols/*` returns `501`. | -| REALPLAN-007-F | DOING | 2026-04-19: Updating runtime proof so the live host resolves durable affected-symbol services and the raw-ingest path can be verified against persisted observation/symbol storage. | +| REALPLAN-007-F | DONE | 2026-04-19: Updated runtime proof so the live host resolves durable affected-symbol services and verified the slice with the targeted xUnit helper (`UnsupportedRuntimeWiringTests`). | +| REALPLAN-007-G | DONE | 2026-04-19: Extended live-host proof so `/jobs` and `/internal/orch/*` resolve durable PostgreSQL-backed runtime services outside `Testing`, then verified the slice with the focused `UnsupportedRuntimeWiringTests` run (`Total: 5, Failed: 0`) after rebuilding the test host assembly. | | NOMOCK-004 | DONE | 2026-04-18: Extended `UnsupportedRuntimeWiringTests` to prove non-testing mirror import succeeds against a real staged bundle, exposes imported mirror artifacts, and fails cleanly on checksum mismatch. | | NOMOCK-026 | DONE | 2026-04-19: Extended mirror import tests and options tests so the live importer only accepts bundle and trust-root paths under the configured `Mirror.ImportRoot`. Verified with focused Concelier rebuilds plus direct xUnit DLL coverage after `dotnet test --filter` proved unreliable in the current test platform path. | | REALPLAN-007-D | DONE | 2026-04-19: Extended `UnsupportedRuntimeWiringTests` so advisory-source sync compatibility routes also prove explicit `501` behavior when the live host resolves `UnsupportedJobCoordinator`. | +| AUDIT-002-CONCELIER | DONE | 2026-04-19: Added focused audit-emission coverage for mirror-domain creation, job trigger, and advisory-source connectivity checks, verified with `scripts/test-targeted-xunit.ps1` against the Concelier WebService test project. | +| AUDIT-002-CONCELIER-DURABLE | DONE | 2026-04-20: Added `DurableAuditEmissionEndpointTests` proving the real no-stub path: live Concelier `POST /jobs/{jobKind}` emits to Timeline ingest over HTTP, Timeline persists the audit event in PostgreSQL, the event survives a Timeline host restart, and chain verification still passes. Verified with the targeted xUnit helper (`Total: 1, Failed: 0`). | +| CONN-ALIGN-007 | DONE | 2026-04-22: Extended advisory-source catalog/status and built-in job registration coverage for canonical `adobe` and `chromium` runtime pipelines. | diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/UnsupportedRuntimeWiringTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/UnsupportedRuntimeWiringTests.cs index 87105a5e0..b63e98e37 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/UnsupportedRuntimeWiringTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/UnsupportedRuntimeWiringTests.cs @@ -16,9 +16,11 @@ using StellaOps.Concelier.Core.Orchestration; using StellaOps.Concelier.Core.Raw; using StellaOps.Concelier.Core.Signals; using StellaOps.Concelier.Persistence.Postgres.Repositories; +using StellaOps.Concelier.Storage; using StellaOps.Replay.Core.FeedSnapshot; using StellaOps.Concelier.WebService.Extensions; using StellaOps.Concelier.WebService.Services; +using StellaOps.Concelier.WebService.Tests.Fixtures; using Moq; using System.Net.Http.Json; using System.Security.Cryptography; @@ -32,81 +34,57 @@ namespace StellaOps.Concelier.WebService.Tests; public sealed class UnsupportedRuntimeWiringTests { [Fact] - public void DevelopmentHost_ResolvesUnsupportedRuntimeServices() + public void DevelopmentHost_DoesNotResolveLegacyInMemoryStorageBootstrapper() { using var factory = new UnsupportedRuntimeWebApplicationFactory(); using var scope = factory.Services.CreateScope(); - scope.ServiceProvider.GetRequiredService().Should().BeOfType(); - scope.ServiceProvider.GetRequiredService().Should().BeOfType(); - scope.ServiceProvider.GetRequiredService().Should().BeOfType(); + scope.ServiceProvider.GetService().Should().BeNull(); + } + + [Fact] + public void DevelopmentHost_DefaultsObservationEventTransportToPostgres() + { + using var factory = new UnsupportedRuntimeWebApplicationFactory(); + using var scope = factory.Services.CreateScope(); + + scope.ServiceProvider + .GetRequiredService>() + .Value + .Transport + .Should() + .Be("postgres"); + } + + [Fact] + public void TestingHost_ResolvesLegacyInMemoryStorageOnlyViaExplicitFactoryWiring() + { + using var factory = new ConcelierApplicationFactory() + .WithWebHostBuilder(builder => + { + builder.ConfigureServices(services => + { + services.RemoveAll(); + }); + }); + using var scope = factory.Services.CreateScope(); + + scope.ServiceProvider.GetRequiredService().Should().NotBeNull(); + } + + [Fact] + public void DevelopmentHost_ResolvesDurableRuntimeServices() + { + using var factory = new UnsupportedRuntimeWebApplicationFactory(); + using var scope = factory.Services.CreateScope(); + + scope.ServiceProvider.GetRequiredService().Should().BeOfType(); + scope.ServiceProvider.GetRequiredService().Should().BeOfType(); + scope.ServiceProvider.GetRequiredService().Should().BeOfType(); + scope.ServiceProvider.GetRequiredService().Should().BeOfType(); scope.ServiceProvider.GetRequiredService().Should().BeOfType(); scope.ServiceProvider.GetRequiredService().Should().BeOfType(); - factory.HasJobSchedulerHostedService.Should().BeFalse(); - } - - [Fact] - public async Task DevelopmentHost_JobsEndpoint_ReturnsNotImplemented() - { - using var factory = new UnsupportedRuntimeWebApplicationFactory(); - using var client = factory.CreateClient(); - - var response = await client.GetAsync("/jobs/definitions"); - var body = await response.Content.ReadAsStringAsync(); - - response.StatusCode.Should().Be(System.Net.HttpStatusCode.NotImplemented); - body.Should().Contain("NOT_IMPLEMENTED"); - body.Should().Contain("durable backend implementation"); - } - - [Fact] - public async Task DevelopmentHost_SourceSyncEndpoint_ReturnsNotImplemented() - { - using var factory = new UnsupportedRuntimeWebApplicationFactory(); - using var client = factory.CreateClient(); - using var request = new HttpRequestMessage(HttpMethod.Post, "/api/v1/advisory-sources/nvd/sync"); - request.Headers.Add(Program.TenantHeaderName, "tenant-a"); - - var response = await client.SendAsync(request); - var body = await response.Content.ReadAsStringAsync(); - - response.StatusCode.Should().Be(System.Net.HttpStatusCode.NotImplemented); - body.Should().Contain("NOT_IMPLEMENTED"); - body.Should().Contain("durable backend implementation"); - } - - [Fact] - public async Task DevelopmentHost_BatchSourceSyncEndpoint_ReturnsNotImplemented() - { - using var factory = new UnsupportedRuntimeWebApplicationFactory(); - using var client = factory.CreateClient(); - using var request = new HttpRequestMessage(HttpMethod.Post, "/api/v1/advisory-sources/sync"); - request.Headers.Add(Program.TenantHeaderName, "tenant-a"); - - var response = await client.SendAsync(request); - var body = await response.Content.ReadAsStringAsync(); - - response.StatusCode.Should().Be(System.Net.HttpStatusCode.NotImplemented); - body.Should().Contain("NOT_IMPLEMENTED"); - body.Should().Contain("durable backend implementation"); - } - - [Fact] - public async Task DevelopmentHost_InternalOrchestratorEndpoint_ReturnsNotImplemented() - { - using var factory = new UnsupportedRuntimeWebApplicationFactory(); - using var client = factory.CreateClient(); - using var request = new HttpRequestMessage( - HttpMethod.Get, - $"/internal/orch/commands?connectorId=nvd&runId={Guid.NewGuid()}"); - request.Headers.Add(Program.TenantHeaderName, "tenant-a"); - - var response = await client.SendAsync(request); - var body = await response.Content.ReadAsStringAsync(); - - response.StatusCode.Should().Be(System.Net.HttpStatusCode.NotImplemented); - body.Should().Contain("NOT_IMPLEMENTED"); - body.Should().Contain("orchestrator registry"); + factory.HasJobSchedulerHostedService.Should().BeTrue(); } [Fact] diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/WebServiceEndpointsTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/WebServiceEndpointsTests.cs index 3ea7a28d6..4f79401c6 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/WebServiceEndpointsTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/WebServiceEndpointsTests.cs @@ -27,6 +27,7 @@ using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using StellaOps.Audit.Emission; using StellaOps.Concelier.InMemoryRunner; using StellaOps.Concelier.Documents; using StellaOps.Concelier.Documents.IO; @@ -1660,6 +1661,194 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime Assert.True(importStatusJson.RootElement.GetProperty("hasImport").GetBoolean()); } + [Fact] + public async Task MirrorDomainCreate_AcceptsRegisteredCatalogKeys_WhenPersistedSourceRowsAreMissing() + { + using var temp = new TempDirectory(); + var environment = new Dictionary + { + ["CONCELIER_MIRROR__ENABLED"] = "true", + ["CONCELIER_MIRROR__EXPORTROOT"] = temp.Path, + ["CONCELIER_MIRROR__IMPORTROOT"] = temp.Path, + ["CONCELIER_MIRROR__ACTIVEEXPORTID"] = "latest", + ["CONCELIER_MIRROR__MAXINDEXREQUESTSPERHOUR"] = "5", + ["CONCELIER_MIRROR__MAXDOWNLOADREQUESTSPERHOUR"] = "5" + }; + + using var factory = new ConcelierApplicationFactory(_runner.ConnectionString, environmentOverrides: environment) + .WithWebHostBuilder(builder => + builder.ConfigureTestServices(services => + { + services.RemoveAll(); + services.AddSingleton(_ => new StubSourceRepository(Array.Empty())); + })); + using var client = factory.CreateClient(); + client.DefaultRequestHeaders.Add("X-Stella-Tenant", "test-tenant"); + + var catalogResponse = await client.GetAsync("/api/v1/advisory-sources/catalog"); + Assert.Equal(HttpStatusCode.OK, catalogResponse.StatusCode); + var catalogJson = JsonDocument.Parse(await catalogResponse.Content.ReadAsStringAsync()); + var catalogIds = catalogJson.RootElement + .GetProperty("items") + .EnumerateArray() + .Select(item => item.GetProperty("id").GetString()) + .Where(id => id is not null) + .Cast() + .ToArray(); + Assert.Contains("nvd", catalogIds); + Assert.Contains("osv", catalogIds); + + var createResponse = await client.PostAsJsonAsync( + "/api/v1/advisory-sources/mirror/domains", + new + { + domainId = "fresh-db", + displayName = "Fresh DB", + sourceIds = new[] { "nvd", "osv" }, + exportFormat = "JSON", + rateLimits = new + { + indexRequestsPerHour = 60, + downloadRequestsPerHour = 120 + }, + requireAuthentication = false, + signing = new + { + enabled = false, + algorithm = "HMAC-SHA256", + keyId = string.Empty + } + }); + + var body = await createResponse.Content.ReadAsStringAsync(); + Assert.True( + createResponse.StatusCode == HttpStatusCode.Created, + $"Mirror domain creation should accept registered catalog keys even before source rows are persisted. Got {(int)createResponse.StatusCode}: {body}"); + } + + [Fact] + public async Task MirrorDomainCreate_EmitsAuditEvent() + { + using var temp = new TempDirectory(); + var environment = new Dictionary + { + ["CONCELIER_MIRROR__ENABLED"] = "true", + ["CONCELIER_MIRROR__EXPORTROOT"] = temp.Path, + ["CONCELIER_MIRROR__IMPORTROOT"] = temp.Path, + ["CONCELIER_MIRROR__ACTIVEEXPORTID"] = "latest", + ["CONCELIER_MIRROR__MAXINDEXREQUESTSPERHOUR"] = "5", + ["CONCELIER_MIRROR__MAXDOWNLOADREQUESTSPERHOUR"] = "5" + }; + + using var factory = new ConcelierApplicationFactory(_runner.ConnectionString, environmentOverrides: environment); + factory.AuditEmitter.Clear(); + + using var client = factory.CreateClient(); + client.DefaultRequestHeaders.Add("X-Stella-Tenant", "test-tenant"); + + var createResponse = await client.PostAsJsonAsync( + "/api/v1/advisory-sources/mirror/domains", + new + { + domainId = "audit-domain", + displayName = "Audit Domain", + sourceIds = new[] { "nvd" }, + exportFormat = "JSON", + rateLimits = new + { + indexRequestsPerHour = 60, + downloadRequestsPerHour = 120 + }, + requireAuthentication = false, + signing = new + { + enabled = false, + algorithm = "HMAC-SHA256", + keyId = string.Empty + } + }); + + var body = await createResponse.Content.ReadAsStringAsync(); + Assert.True( + createResponse.StatusCode == HttpStatusCode.Created, + $"Mirror domain creation failed with {(int)createResponse.StatusCode}: {body}"); + + var auditEvent = await factory.AuditEmitter.WaitForEventAsync( + evt => evt.Module == "concelier" + && evt.Action == "create" + && evt.Resource.Type == "mirror_domain", + TimeSpan.FromSeconds(5), + CancellationToken.None); + + Assert.NotNull(auditEvent); + Assert.Equal("mirror_domain", auditEvent!.Resource.Type); + Assert.Equal("audit-domain", auditEvent.Resource.Id); + } + + [Fact] + public async Task JobTrigger_EmitsAuditEvent() + { + var handler = _factory.Services.GetRequiredService(); + _factory.AuditEmitter.Clear(); + + handler.NextResult = JobTriggerResult.Accepted( + new JobRunSnapshot( + Guid.NewGuid(), + "demo", + JobRunStatus.Pending, + DateTimeOffset.UtcNow, + null, + null, + "api", + null, + null, + null, + null, + new Dictionary())); + + using var client = _factory.CreateClient(); + var response = await client.PostAsync("/jobs/test", JsonContent.Create(new JobTriggerRequest())); + Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); + + var auditEvent = await _factory.AuditEmitter.WaitForEventAsync( + evt => evt.Module == "concelier" + && evt.Action == "execute" + && evt.Resource.Type == "job", + TimeSpan.FromSeconds(5), + CancellationToken.None); + + Assert.NotNull(auditEvent); + Assert.Equal("job", auditEvent!.Resource.Type); + Assert.Equal("execute", auditEvent.Action); + } + + [Fact] + public async Task CheckSourceConnectivity_EmitsAuditEvent() + { + _factory.AuditEmitter.Clear(); + + using var client = _factory.CreateClient(); + client.DefaultRequestHeaders.Add("X-Stella-Tenant", "test-tenant"); + + var response = await client.PostAsync("/api/v1/advisory-sources/nvd/check", content: null); + + var body = await response.Content.ReadAsStringAsync(); + Assert.True( + response.StatusCode == HttpStatusCode.OK, + $"Source connectivity check failed with {(int)response.StatusCode}: {body}"); + + var auditEvent = await _factory.AuditEmitter.WaitForEventAsync( + evt => evt.Module == "concelier" + && evt.Action == "verify" + && evt.Resource.Type == "advisory_source", + TimeSpan.FromSeconds(5), + CancellationToken.None); + + Assert.NotNull(auditEvent); + Assert.Equal("advisory_source", auditEvent!.Resource.Type); + Assert.Equal("verify", auditEvent.Action); + } + private static async Task CreateMirrorDomainAsync( HttpClient client, string domainId, @@ -2204,6 +2393,7 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime private readonly Action? _authorityConfigure; private readonly IDictionary _additionalPreviousEnvironment = new Dictionary(StringComparer.OrdinalIgnoreCase); public CollectingLoggerProvider LoggerProvider { get; } = new(); + public RecordingAuditEventEmitter AuditEmitter { get; } = new(); public ConcelierApplicationFactory( string connectionString, @@ -2315,6 +2505,8 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime builder.ConfigureServices(services => { + StellaOps.Concelier.Storage.LegacyServiceCollectionExtensions.AddInMemoryStorage(services); + // Keep ConcelierDataSource - tests now use a real PostgreSQL database via Docker. // The database is expected to run on localhost:5432 with database=concelier_test. @@ -2439,6 +2631,8 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime services.AddSingleton(); services.RemoveAll(); services.AddSingleton(); + services.RemoveAll(); + services.AddSingleton(AuditEmitter); services.AddSingleton(); @@ -3875,37 +4069,79 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime } } + private sealed class RecordingAuditEventEmitter : IAuditEventEmitter + { + private readonly object gate = new(); + private readonly List events = []; + + public IReadOnlyList Events + { + get + { + lock (gate) + { + return events.ToList(); + } + } + } + + public Task EmitAsync(AuditEventPayload auditEvent, CancellationToken cancellationToken) + { + lock (gate) + { + events.Add(auditEvent); + } + + return Task.CompletedTask; + } + + public void Clear() + { + lock (gate) + { + events.Clear(); + } + } + + public async Task WaitForEventAsync( + Func predicate, + TimeSpan timeout, + CancellationToken cancellationToken) + { + var deadline = DateTime.UtcNow + timeout; + while (DateTime.UtcNow < deadline) + { + cancellationToken.ThrowIfCancellationRequested(); + + lock (gate) + { + var match = events.FirstOrDefault(predicate); + if (match is not null) + { + return match; + } + } + + await Task.Delay(50, cancellationToken); + } + + return null; + } + } + private sealed class StubSourceRepository : ISourceRepository { - private readonly List sources = - [ - new SourceEntity - { - Id = Guid.Parse("0e94e643-4045-4af8-b0c4-f4f04f769279"), - Key = "nvd", - Name = "NVD", - SourceType = "nvd", - Url = "https://nvd.nist.gov", - Priority = 100, - Enabled = true, - Config = """{"syncIntervalMinutes":360,"localPath":"/data/mirrors/nvd"}""", - CreatedAt = DateTimeOffset.Parse("2026-02-18T00:00:00Z", CultureInfo.InvariantCulture), - UpdatedAt = DateTimeOffset.Parse("2026-02-19T00:00:00Z", CultureInfo.InvariantCulture), - }, - new SourceEntity - { - Id = Guid.Parse("db66a4a1-b0bf-4bb4-a9cb-b9ea2e057613"), - Key = "osv", - Name = "OSV", - SourceType = "osv", - Url = "https://osv.dev", - Priority = 90, - Enabled = true, - Config = """{"syncIntervalMinutes":360,"localPath":"/data/mirrors/osv"}""", - CreatedAt = DateTimeOffset.Parse("2026-02-18T00:00:00Z", CultureInfo.InvariantCulture), - UpdatedAt = DateTimeOffset.Parse("2026-02-19T00:00:00Z", CultureInfo.InvariantCulture), - } - ]; + private readonly List sources; + + public StubSourceRepository() + : this(CreateDefaultSources()) + { + } + + public StubSourceRepository(IEnumerable seedSources) + { + sources = seedSources.ToList(); + } public Task UpsertAsync(SourceEntity source, CancellationToken cancellationToken = default) { @@ -3937,6 +4173,36 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime .ToList(); return Task.FromResult>(items); } + + private static IReadOnlyList CreateDefaultSources() => + [ + new SourceEntity + { + Id = Guid.Parse("0e94e643-4045-4af8-b0c4-f4f04f769279"), + Key = "nvd", + Name = "NVD", + SourceType = "nvd", + Url = "https://nvd.nist.gov", + Priority = 100, + Enabled = true, + Config = """{"syncIntervalMinutes":360,"localPath":"/data/mirrors/nvd"}""", + CreatedAt = DateTimeOffset.Parse("2026-02-18T00:00:00Z", CultureInfo.InvariantCulture), + UpdatedAt = DateTimeOffset.Parse("2026-02-19T00:00:00Z", CultureInfo.InvariantCulture), + }, + new SourceEntity + { + Id = Guid.Parse("db66a4a1-b0bf-4bb4-a9cb-b9ea2e057613"), + Key = "osv", + Name = "OSV", + SourceType = "osv", + Url = "https://osv.dev", + Priority = 90, + Enabled = true, + Config = """{"syncIntervalMinutes":360,"localPath":"/data/mirrors/osv"}""", + CreatedAt = DateTimeOffset.Parse("2026-02-18T00:00:00Z", CultureInfo.InvariantCulture), + UpdatedAt = DateTimeOffset.Parse("2026-02-19T00:00:00Z", CultureInfo.InvariantCulture), + } + ]; } private sealed class InMemoryMirrorBundleImportStore : IMirrorBundleImportStore diff --git a/src/Concelier/__Tests/StellaOps.Excititor.Attestation.Tests/VexAttestationServiceCollectionExtensionsTests.cs b/src/Concelier/__Tests/StellaOps.Excititor.Attestation.Tests/VexAttestationServiceCollectionExtensionsTests.cs new file mode 100644 index 000000000..27f444645 --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Excititor.Attestation.Tests/VexAttestationServiceCollectionExtensionsTests.cs @@ -0,0 +1,23 @@ +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Excititor.Attestation.Extensions; +using StellaOps.Excititor.Attestation.Verification; +using Xunit; + +namespace StellaOps.Excititor.Attestation.Tests; + +public sealed class VexAttestationServiceCollectionExtensionsTests +{ + [Fact] + public void AddVexAttestation_ResolvesVerifierWithoutTransparencyClient() + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddOptions(); + services.AddVexAttestation(); + + using var serviceProvider = services.BuildServiceProvider(); + var verifier = serviceProvider.GetRequiredService(); + + Assert.NotNull(verifier); + } +} diff --git a/src/Concelier/__Tests/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests/DependencyInjection/CiscoConnectorServiceCollectionExtensionsTests.cs b/src/Concelier/__Tests/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests/DependencyInjection/CiscoConnectorServiceCollectionExtensionsTests.cs new file mode 100644 index 000000000..bf09005a6 --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests/DependencyInjection/CiscoConnectorServiceCollectionExtensionsTests.cs @@ -0,0 +1,34 @@ +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration; +using StellaOps.Excititor.Connectors.Cisco.CSAF.DependencyInjection; +using System.Linq; +using System.Net.Http; + +namespace StellaOps.Excititor.Connectors.Cisco.CSAF.Tests.DependencyInjection; + +public sealed class CiscoConnectorServiceCollectionExtensionsTests +{ + [Fact] + public void AddCiscoCsafConnector_ConfiguresBrowserSafeJsonClient() + { + var services = new ServiceCollection(); + services.AddCiscoCsafConnector(); + + using var provider = services.BuildServiceProvider(); + var factory = provider.GetRequiredService(); + using var client = factory.CreateClient(CiscoConnectorOptions.HttpClientName); + + client.DefaultRequestVersion.Should().Be(new Version(1, 1)); + client.DefaultVersionPolicy.Should().Be(HttpVersionPolicy.RequestVersionExact); + client.DefaultRequestHeaders.UserAgent.ToString().Should().Contain("Mozilla/5.0"); + client.DefaultRequestHeaders.AcceptLanguage.ToString().Should().Contain("en-US"); + client.DefaultRequestHeaders.Accept.Select(static header => header.MediaType).Should().Contain(new[] + { + "application/csaf+json", + "application/json", + "text/plain", + "*/*", + }); + } +} diff --git a/src/Concelier/__Tests/StellaOps.Excititor.Core.Tests/Verification/VexVerificationRuntimeRegistrationTests.cs b/src/Concelier/__Tests/StellaOps.Excititor.Core.Tests/Verification/VexVerificationRuntimeRegistrationTests.cs new file mode 100644 index 000000000..3c14955ef --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Excititor.Core.Tests/Verification/VexVerificationRuntimeRegistrationTests.cs @@ -0,0 +1,198 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Excititor.Core.Verification; +using StellaOps.TestKit; +using StellaOps.TestKit.Fixtures; +using Xunit; + +namespace StellaOps.Excititor.Core.Tests.Verification; + +public sealed class VexVerificationRuntimeRegistrationTests +{ + [Fact] + public void AddVexSignatureVerification_Disabled_ResolvesNoopVerifierWithoutImplicitRuntimeFallbacks() + { + var services = CreateBaseServices(); + + services.AddVexSignatureVerification(BuildConfiguration()); + + using var provider = services.BuildServiceProvider(); + + var verifier = provider.GetRequiredService(); + + verifier.GetType().Name.Should().Be("NoopVexSignatureVerifierV2"); + provider.GetService().Should().BeNull(); + } + + [Fact] + public void AddVexSignatureVerification_EnabledWithoutIssuerDirectoryUrl_ThrowsInsteadOfUsingInMemoryFallback() + { + var services = CreateBaseServices(); + var configuration = BuildConfiguration(new Dictionary + { + [$"{VexSignatureVerifierOptions.SectionName}:Enabled"] = "true" + }); + + services.AddVexSignatureVerification(configuration); + + using var provider = services.BuildServiceProvider(); + + var act = () => provider.GetRequiredService(); + + act.Should() + .Throw() + .WithMessage("*IssuerDirectory:ServiceUrl*") + .WithMessage("*in-memory/offline issuer-directory fallback*"); + } + + [Fact] + public void AddVexSignatureVerification_EnabledWithIssuerDirectoryUrl_UsesHttpIssuerDirectoryWithoutImplicitCache() + { + var services = CreateBaseServices(); + var configuration = BuildConfiguration(new Dictionary + { + [$"{VexSignatureVerifierOptions.SectionName}:Enabled"] = "true", + [$"{VexSignatureVerifierOptions.SectionName}:IssuerDirectory:ServiceUrl"] = "https://issuer-directory.stella.local" + }); + + services.AddVexSignatureVerification(configuration); + + using var provider = services.BuildServiceProvider(); + + provider.GetRequiredService() + .Should() + .BeOfType(); + provider.GetRequiredService() + .Should() + .BeOfType(); + provider.GetService().Should().BeNull(); + } + + [Fact] + public async Task AddVexSignatureVerificationWithValkey_RegistersRealValkeyCacheService() + { + var services = CreateBaseServices(); + var configuration = BuildConfiguration(new Dictionary + { + [$"{VexSignatureVerifierOptions.SectionName}:Enabled"] = "true", + [$"{VexSignatureVerifierOptions.SectionName}:IssuerDirectory:ServiceUrl"] = "https://issuer-directory.stella.local" + }); + + services.AddVexSignatureVerificationWithValkey( + configuration, + "127.0.0.1:6379,defaultDatabase=5"); + + await using var provider = services.BuildServiceProvider(); + + provider.GetRequiredService() + .Should() + .BeOfType(); + provider.GetRequiredService() + .Should() + .BeOfType(); + } + + private static ServiceCollection CreateBaseServices() + { + var services = new ServiceCollection(); + services.AddLogging(); + return services; + } + + private static IConfiguration BuildConfiguration( + IDictionary? overrides = null) + { + var values = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [$"{VexSignatureVerifierOptions.SectionName}:Enabled"] = "false" + }; + + if (overrides is not null) + { + foreach (var (key, value) in overrides) + { + values[key] = value; + } + } + + return new ConfigurationBuilder() + .AddInMemoryCollection(values) + .Build(); + } +} + +public sealed class ValkeyVerificationCacheServiceRuntimeTests : IClassFixture +{ + private readonly ValkeyFixture _valkeyFixture; + + public ValkeyVerificationCacheServiceRuntimeTests(ValkeyFixture valkeyFixture) + { + _valkeyFixture = valkeyFixture; + _valkeyFixture.IsolationMode = ValkeyIsolationMode.DatabasePerTest; + } + + [Trait("Category", TestCategories.Integration)] + [Fact] + public async Task SetGetAndInvalidateByIssuer_UsesRealValkeyStorage() + { + await using var session = await _valkeyFixture.CreateSessionAsync( + nameof(SetGetAndInvalidateByIssuer_UsesRealValkeyStorage)); + await using var cache = new ValkeyVerificationCacheService( + _valkeyFixture.ConnectionString, + NullLogger.Instance, + session.DatabaseIndex); + + var issuerOneResult = VexSignatureVerificationResult.Success( + "sha256:issuer-one-a", + VerificationMethod.Dsse, + issuerId: "issuer-1"); + var issuerOneSecondResult = VexSignatureVerificationResult.Success( + "sha256:issuer-one-b", + VerificationMethod.Dsse, + issuerId: "issuer-1"); + var issuerTwoResult = VexSignatureVerificationResult.Success( + "sha256:issuer-two-a", + VerificationMethod.Dsse, + issuerId: "issuer-2"); + + await cache.SetAsync("entry-1", issuerOneResult, TimeSpan.FromMinutes(10), CancellationToken.None); + await cache.SetAsync("entry-2", issuerOneSecondResult, TimeSpan.FromMinutes(10), CancellationToken.None); + await cache.SetAsync("entry-3", issuerTwoResult, TimeSpan.FromMinutes(10), CancellationToken.None); + + (await session.Database.StringGetAsync("excititor:vex-verification:entry-1")) + .IsNullOrEmpty + .Should() + .BeFalse("the cache should persist serialized entries in Valkey"); + + var cachedEntryOne = await cache.GetAsync("entry-1", CancellationToken.None); + var cachedEntryThree = await cache.GetAsync("entry-3", CancellationToken.None); + + cachedEntryOne.Should().NotBeNull(); + cachedEntryOne!.IssuerId.Should().Be("issuer-1"); + cachedEntryThree.Should().NotBeNull(); + cachedEntryThree!.IssuerId.Should().Be("issuer-2"); + + await cache.InvalidateByIssuerAsync("issuer-1", CancellationToken.None); + + (await cache.GetAsync("entry-1", CancellationToken.None)).Should().BeNull(); + (await cache.GetAsync("entry-2", CancellationToken.None)).Should().BeNull(); + var remaining = await cache.GetAsync("entry-3", CancellationToken.None); + remaining.Should().NotBeNull(); + remaining!.IssuerId.Should().Be("issuer-2"); + + (await session.Database.StringGetAsync("excititor:vex-verification:entry-1")) + .IsNullOrEmpty + .Should() + .BeTrue("issuer invalidation should remove the entry from Valkey"); + (await session.Database.StringGetAsync("excititor:vex-verification:entry-3")) + .IsNullOrEmpty + .Should() + .BeFalse("other issuers must remain cached"); + } +} diff --git a/src/Concelier/__Tests/StellaOps.Excititor.Core.UnitTests/VexNormalizationServiceCollectionExtensionsTests.cs b/src/Concelier/__Tests/StellaOps.Excititor.Core.UnitTests/VexNormalizationServiceCollectionExtensionsTests.cs new file mode 100644 index 000000000..750551d61 --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Excititor.Core.UnitTests/VexNormalizationServiceCollectionExtensionsTests.cs @@ -0,0 +1,108 @@ +using System.Collections.Immutable; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Excititor.Core.Storage; +using Xunit; + +namespace StellaOps.Excititor.Core.UnitTests; + +public sealed class VexNormalizationServiceCollectionExtensionsTests +{ + [Fact] + public async Task AddVexNormalization_RegistersRouter_AndUsesMatchingNormalizer() + { + var services = new ServiceCollection(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddVexNormalization(); + + using var serviceProvider = services.BuildServiceProvider(); + var providerStore = serviceProvider.GetRequiredService(); + await providerStore.SaveAsync( + new VexProvider("excititor:redhat", "Red Hat CSAF", VexProviderKind.Distro), + CancellationToken.None); + + var router = serviceProvider.GetRequiredService(); + var batch = await router.NormalizeAsync( + CreateDocument("excititor:redhat", VexDocumentFormat.Csaf), + CancellationToken.None); + + var claim = Assert.Single(batch.Claims); + Assert.Equal("CVE-CSAF-0001", claim.VulnerabilityId); + Assert.Equal("excititor:redhat", claim.ProviderId); + Assert.Equal(VexClaimStatus.NotAffected, claim.Status); + } + + [Fact] + public async Task AddVexNormalization_ThrowsWhenNoNormalizerCanHandleDocument() + { + var services = new ServiceCollection(); + services.AddSingleton(); + services.AddSingleton(); + services.AddVexNormalization(); + + using var serviceProvider = services.BuildServiceProvider(); + var router = serviceProvider.GetRequiredService(); + + var exception = await Assert.ThrowsAsync(async () => + await router.NormalizeAsync( + CreateDocument("excititor:redhat", VexDocumentFormat.Csaf), + CancellationToken.None)); + + Assert.Contains("Csaf", exception.Message, StringComparison.Ordinal); + } + + private static VexRawDocument CreateDocument(string providerId, VexDocumentFormat format) + { + return new VexRawDocument( + providerId, + format, + new Uri("https://mirror.example.test/document.json"), + DateTimeOffset.Parse("2026-04-21T16:31:51Z"), + "sha256:test-document", + new byte[] { 0x7b, 0x7d }, + ImmutableDictionary.Empty); + } + + private sealed class StubCsafNormalizer : IVexNormalizer + { + public string Format => VexDocumentFormat.Csaf.ToString().ToLowerInvariant(); + + public bool CanHandle(VexRawDocument document) => document.Format == VexDocumentFormat.Csaf; + + public ValueTask NormalizeAsync(VexRawDocument document, VexProvider provider, CancellationToken cancellationToken) + { + var claim = new VexClaim( + "CVE-CSAF-0001", + provider.Id, + new VexProduct("pkg:test/redhat", "Red Hat test package"), + VexClaimStatus.NotAffected, + new VexClaimDocument(document.Format, document.Digest, document.SourceUri, revision: "1"), + document.RetrievedAt, + document.RetrievedAt); + + return ValueTask.FromResult(new VexClaimBatch(document, [claim], ImmutableDictionary.Empty)); + } + } + + private sealed class StubOpenVexNormalizer : IVexNormalizer + { + public string Format => VexDocumentFormat.OpenVex.ToString().ToLowerInvariant(); + + public bool CanHandle(VexRawDocument document) => document.Format == VexDocumentFormat.OpenVex; + + public ValueTask NormalizeAsync(VexRawDocument document, VexProvider provider, CancellationToken cancellationToken) + { + var claim = new VexClaim( + "CVE-OPENVEX-0001", + provider.Id, + new VexProduct("pkg:test/openvex", "OpenVEX test package"), + VexClaimStatus.Fixed, + new VexClaimDocument(document.Format, document.Digest, document.SourceUri, revision: "1"), + document.RetrievedAt, + document.RetrievedAt); + + return ValueTask.FromResult(new VexClaimBatch(document, [claim], ImmutableDictionary.Empty)); + } + } +} diff --git a/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/PostgresAppendOnlyCheckpointStoreTests.cs b/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/PostgresAppendOnlyCheckpointStoreTests.cs new file mode 100644 index 000000000..362af0e11 --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/PostgresAppendOnlyCheckpointStoreTests.cs @@ -0,0 +1,101 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Core.Storage; +using StellaOps.Excititor.Persistence.Postgres; +using StellaOps.Excititor.Persistence.Postgres.Repositories; +using StellaOps.Infrastructure.Postgres.Options; +using StellaOps.TestKit; +using Xunit; + +namespace StellaOps.Excititor.Persistence.Tests; + +[Collection(ExcititorPostgresCollection.Name)] +public sealed class PostgresAppendOnlyCheckpointStoreTests : IAsyncLifetime +{ + private readonly ExcititorPostgresFixture _fixture; + private readonly PostgresAppendOnlyCheckpointStore _store; + private readonly ExcititorDataSource _dataSource; + + public PostgresAppendOnlyCheckpointStoreTests(ExcititorPostgresFixture fixture) + { + _fixture = fixture; + var options = Options.Create(new PostgresOptions + { + ConnectionString = fixture.ConnectionString, + SchemaName = fixture.SchemaName, + AutoMigrate = false + }); + + _dataSource = new ExcititorDataSource(options, NullLogger.Instance); + _store = new PostgresAppendOnlyCheckpointStore(_dataSource, NullLogger.Instance); + } + + public async ValueTask InitializeAsync() + { + await _fixture.Fixture.RunMigrationsFromAssemblyAsync( + typeof(ExcititorDataSource).Assembly, + moduleName: "Excititor", + resourcePrefix: "Migrations", + cancellationToken: CancellationToken.None); + + await _fixture.TruncateAllTablesAsync(); + } + + public async ValueTask DisposeAsync() + { + await _dataSource.DisposeAsync(); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task AppendAsync_HeartbeatMaterializesCheckpointState() + { + var runId = Guid.NewGuid(); + var timestamp = DateTimeOffset.UtcNow; + + var result = await _store.AppendAsync( + "public", + "excititor:redhat", + CheckpointMutation.Heartbeat(runId, timestamp, cursor: "cursor-1"), + CancellationToken.None); + + result.Success.Should().BeTrue(); + result.WasDuplicate.Should().BeFalse(); + result.CurrentState.ConnectorId.Should().Be("excititor:redhat"); + result.CurrentState.Cursor.Should().Be("cursor-1"); + result.CurrentState.LastRunId.Should().Be(runId); + result.CurrentState.LastMutationType.Should().Be(CheckpointMutationType.Heartbeat); + result.CurrentState.LatestSequenceNumber.Should().BeGreaterThan(0); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task AppendAsync_FailedMutationUpdatesRetryWindow() + { + var runId = Guid.NewGuid(); + var timestamp = DateTimeOffset.UtcNow; + + await _store.AppendAsync( + "public", + "excititor:redhat", + CheckpointMutation.Failed( + runId, + timestamp, + errorCode: "INTERNAL_ERROR", + errorMessage: "boom", + retryAfterSeconds: 120, + cursor: "cursor-2"), + CancellationToken.None); + + var state = await _store.GetCurrentStateAsync("public", "excititor:redhat", CancellationToken.None); + + state.Should().NotBeNull(); + state!.LastMutationType.Should().Be(CheckpointMutationType.Failed); + state.LastErrorCode.Should().Be("INTERNAL_ERROR"); + state.Cursor.Should().Be("cursor-2"); + state.FailureCount.Should().Be(1); + state.NextEligibleRun.Should().NotBeNull(); + state.NextEligibleRun.Should().BeOnOrAfter(timestamp.AddSeconds(120).AddSeconds(-1)); + } +} diff --git a/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/PostgresVexEvidenceLinkStoreTests.cs b/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/PostgresVexEvidenceLinkStoreTests.cs new file mode 100644 index 000000000..1128e7040 --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/PostgresVexEvidenceLinkStoreTests.cs @@ -0,0 +1,143 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Core.Evidence; +using StellaOps.Excititor.Persistence.Postgres; +using StellaOps.Excititor.Persistence.Postgres.Repositories; +using StellaOps.Infrastructure.Postgres.Options; +using StellaOps.TestKit; +using System.Collections.Immutable; +using Xunit; + +namespace StellaOps.Excititor.Persistence.Tests; + +[Collection(ExcititorPostgresCollection.Name)] +public sealed class PostgresVexEvidenceLinkStoreTests : IAsyncLifetime +{ + private readonly ExcititorPostgresFixture _fixture; + private readonly PostgresVexEvidenceLinkStore _store; + private readonly ExcititorDataSource _dataSource; + + public PostgresVexEvidenceLinkStoreTests(ExcititorPostgresFixture fixture) + { + _fixture = fixture; + + var options = Options.Create(new PostgresOptions + { + ConnectionString = fixture.ConnectionString, + SchemaName = ExcititorDataSource.DefaultSchemaName, + AutoMigrate = false + }); + + _dataSource = new ExcititorDataSource(options, NullLogger.Instance); + _store = new PostgresVexEvidenceLinkStore(_dataSource, NullLogger.Instance); + } + + public async ValueTask InitializeAsync() + { + await _fixture.TruncateAllTablesAsync(); + } + + public async ValueTask DisposeAsync() + { + await _dataSource.DisposeAsync(); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task SaveAndGetAsync_RoundTripsDurableEvidenceLink() + { + var link = CreateLink( + linkId: "link-001", + vexEntryId: "vex:entry:001", + confidence: 0.95, + linkedAt: new DateTimeOffset(2026, 4, 20, 9, 30, 0, TimeSpan.Zero)); + + await _store.SaveAsync(link, CancellationToken.None); + + var fetched = await _store.GetAsync("link-001", CancellationToken.None); + + fetched.Should().NotBeNull(); + fetched!.LinkId.Should().Be("link-001"); + fetched.VexEntryId.Should().Be("vex:entry:001"); + fetched.EvidenceType.Should().Be(EvidenceType.BinaryDiff); + fetched.EvidenceUri.Should().Be("oci://registry/evidence@sha256:link-001"); + fetched.Justification.Should().Be(VexJustification.CodeNotReachable); + fetched.Metadata.Should().Contain(new KeyValuePair("source", "integration-test")); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task GetByVexEntryAsync_OrdersByConfidenceThenLinkedAtThenLinkId() + { + await _store.SaveAsync(CreateLink("link-c", "vex:entry:order", 0.70, new DateTimeOffset(2026, 4, 20, 9, 0, 2, TimeSpan.Zero)), CancellationToken.None); + await _store.SaveAsync(CreateLink("link-b", "vex:entry:order", 0.90, new DateTimeOffset(2026, 4, 20, 9, 0, 1, TimeSpan.Zero)), CancellationToken.None); + await _store.SaveAsync(CreateLink("link-a", "vex:entry:order", 0.90, new DateTimeOffset(2026, 4, 20, 9, 0, 0, TimeSpan.Zero)), CancellationToken.None); + + var links = await _store.GetByVexEntryAsync("vex:entry:order", CancellationToken.None); + + links.Select(link => link.LinkId).Should().Equal("link-a", "link-b", "link-c"); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task SaveAsync_DuplicateLinkId_IsIdempotent() + { + var original = CreateLink("link-dup", "vex:entry:dup", 0.50, new DateTimeOffset(2026, 4, 20, 10, 0, 0, TimeSpan.Zero)); + var conflicting = CreateLink("link-dup", "vex:entry:dup", 0.99, new DateTimeOffset(2026, 4, 20, 11, 0, 0, TimeSpan.Zero)); + + await _store.SaveAsync(original, CancellationToken.None); + await _store.SaveAsync(conflicting, CancellationToken.None); + + var fetched = await _store.GetAsync("link-dup", CancellationToken.None); + var links = await _store.GetByVexEntryAsync("vex:entry:dup", CancellationToken.None); + + fetched.Should().NotBeNull(); + fetched!.Confidence.Should().Be(0.50); + fetched.LinkedAt.Should().Be(original.LinkedAt); + links.Should().HaveCount(1); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task DeleteAsync_RemovesPersistedLink() + { + await _store.SaveAsync(CreateLink("link-delete", "vex:entry:delete", 0.80, new DateTimeOffset(2026, 4, 20, 12, 0, 0, TimeSpan.Zero)), CancellationToken.None); + + await _store.DeleteAsync("link-delete", CancellationToken.None); + + var fetched = await _store.GetAsync("link-delete", CancellationToken.None); + var links = await _store.GetByVexEntryAsync("vex:entry:delete", CancellationToken.None); + + fetched.Should().BeNull(); + links.Should().BeEmpty(); + } + + private static VexEvidenceLink CreateLink( + string linkId, + string vexEntryId, + double confidence, + DateTimeOffset linkedAt) + { + return new VexEvidenceLink + { + LinkId = linkId, + VexEntryId = vexEntryId, + EvidenceType = EvidenceType.BinaryDiff, + EvidenceUri = $"oci://registry/evidence@sha256:{linkId}", + EnvelopeDigest = $"sha256:{linkId}", + PredicateType = "stellaops.binarydiff.v1", + Confidence = confidence, + Justification = VexJustification.CodeNotReachable, + EvidenceCreatedAt = linkedAt.AddMinutes(-10), + LinkedAt = linkedAt, + SignerIdentity = "issuer:test", + RekorLogIndex = "rekor-42", + SignatureValidated = true, + Metadata = ImmutableDictionary.Empty + .Add("source", "integration-test") + .Add("stage", "runtime") + }; + } +} diff --git a/src/Concelier/__Tests/StellaOps.Excititor.Worker.Tests/BuiltInVexProviderDefaultsTests.cs b/src/Concelier/__Tests/StellaOps.Excititor.Worker.Tests/BuiltInVexProviderDefaultsTests.cs new file mode 100644 index 000000000..b483383c9 --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Excititor.Worker.Tests/BuiltInVexProviderDefaultsTests.cs @@ -0,0 +1,52 @@ +using FluentAssertions; +using StellaOps.Excititor.Worker.Options; + +namespace StellaOps.Excititor.Worker.Tests; + +public sealed class BuiltInVexProviderDefaultsTests +{ + [Trait("Category", "Unit")] + [Fact] + public void SeedPublicDefaults_AddsPublicMirrorProviders_WhenNoProvidersConfigured() + { + var providers = new List(); + + StellaOps.Excititor.Worker.Options.BuiltInVexProviderDefaults.SeedPublicDefaults(providers); + + providers.Select(static provider => provider.ProviderId) + .Should() + .Equal( + "excititor:redhat", + "excititor:ubuntu", + "excititor:oracle", + "excititor:cisco"); + + providers.Select(static provider => provider.InitialDelay) + .Should() + .Equal( + TimeSpan.FromMinutes(5), + TimeSpan.FromMinutes(7), + TimeSpan.FromMinutes(9), + TimeSpan.FromMinutes(11)); + } + + [Trait("Category", "Unit")] + [Fact] + public void SeedPublicDefaults_LeavesExplicitConfigurationUntouched() + { + var providers = new List + { + new() + { + ProviderId = "excititor:redhat", + InitialDelay = TimeSpan.FromMinutes(1), + }, + }; + + StellaOps.Excititor.Worker.Options.BuiltInVexProviderDefaults.SeedPublicDefaults(providers); + + providers.Should().HaveCount(1); + providers[0].ProviderId.Should().Be("excititor:redhat"); + providers[0].InitialDelay.Should().Be(TimeSpan.FromMinutes(1)); + } +}