diff --git a/.gitignore b/.gitignore index 6e11a0b9..86af3a96 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,5 @@ TestResults/ seed-data/ics-cisa/*.csv seed-data/ics-cisa/*.xlsx seed-data/ics-cisa/*.sha256 +seed-data/cert-bund/**/*.json +seed-data/cert-bund/**/*.sha256 diff --git a/SPRINTS.md b/SPRINTS.md index b4cfc435..9365279e 100644 --- a/SPRINTS.md +++ b/SPRINTS.md @@ -105,8 +105,8 @@ | Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Feedser.Merge/TASKS.md | DONE (2025-10-15) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-005 | Connector coordination for new advisory fields
GHSA/NVD/OSV connectors now ship description, CWE, and canonical metric data with refreshed fixtures; merge coordination log updated and exporters notified. | | Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Feedser.Exporter.Json/TASKS.md | DONE (2025-10-15) | Team Exporters – JSON | FEEDEXPORT-JSON-04-001 | Surface new advisory fields in JSON exporter
Update schemas/offline bundle + fixtures once model/core parity lands.
2025-10-15: `dotnet test src/StellaOps.Feedser.Exporter.Json.Tests` validated canonical metric/CWE emission. | | Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Feedser.Exporter.TrivyDb/TASKS.md | DONE (2025-10-15) | Team Exporters – Trivy DB | FEEDEXPORT-TRIVY-04-001 | Propagate new advisory fields into Trivy DB package
Extend Bolt builder, metadata, and regression tests for the expanded schema.
2025-10-15: `dotnet test src/StellaOps.Feedser.Exporter.TrivyDb.Tests` confirmed canonical metric/CWE propagation. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | TODO | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-004 | Harden CVSS fallback so canonical metric ids persist when GitHub omits vectors; extend fixtures and document severity precedence hand-off to Merge. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Feedser.Source.Osv/TASKS.md | TODO | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-04-005 | Map OSV advisories lacking CVSS vectors to canonical metric ids/notes and document CWE provenance quirks; schedule parity fixture updates. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | DONE (2025-10-16) | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-004 | Harden CVSS fallback so canonical metric ids persist when GitHub omits vectors; extend fixtures and document severity precedence hand-off to Merge. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Feedser.Source.Osv/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-04-005 | Map OSV advisories lacking CVSS vectors to canonical metric ids/notes and document CWE provenance quirks; schedule parity fixture updates. | | Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Core/TASKS.md | DONE (2025-10-15) | Team Vexer Core & Policy | VEXER-CORE-01-001 | Stand up canonical VEX claim/consensus records with deterministic serializers so Storage/Exports share a stable contract. | | Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Core/TASKS.md | DONE (2025-10-15) | Team Vexer Core & Policy | VEXER-CORE-01-002 | Implement trust-weighted consensus resolver with baseline policy weights, justification gates, telemetry output, and majority/tie handling. | | Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Core/TASKS.md | DONE (2025-10-15) | Team Vexer Core & Policy | VEXER-CORE-01-003 | Publish shared connector/exporter/attestation abstractions and deterministic query signature utilities for cache/attestation workflows. | @@ -116,29 +116,35 @@ | Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Policy/TASKS.md | DONE (2025-10-16) | Team Vexer Policy | VEXER-POLICY-01-004 | Implement YAML/JSON schema validation and deterministic diagnostics for operator bundles. | | Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Policy/TASKS.md | DONE (2025-10-16) | Team Vexer Policy | VEXER-POLICY-01-005 | Add policy change tracking, snapshot digests, and telemetry/logging hooks. | | Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Storage.Mongo/TASKS.md | DONE (2025-10-15) | Team Vexer Storage | VEXER-STORAGE-01-001 | Mongo mapping registry plus raw/export entities and DI extensions in place. | -| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Storage.Mongo/TASKS.md | TODO | Team Vexer Storage | VEXER-STORAGE-01-004 | Build provider/consensus/cache class maps and related collections. | +| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Storage.Mongo/TASKS.md | DONE (2025-10-16) | Team Vexer Storage | VEXER-STORAGE-01-004 | Build provider/consensus/cache class maps and related collections. | | Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Export/TASKS.md | DONE (2025-10-15) | Team Vexer Export | VEXER-EXPORT-01-001 | Export engine delivers cache lookup, manifest creation, and policy integration. | -| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Export/TASKS.md | TODO | Team Vexer Export | VEXER-EXPORT-01-004 | Connect export engine to attestation client and persist Rekor metadata. | -| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Attestation/TASKS.md | TODO | Team Vexer Attestation | VEXER-ATTEST-01-001 | Implement in-toto predicate + DSSE builder providing envelopes for export attestation. | -| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Connectors.Abstractions/TASKS.md | TODO | Team Vexer Connectors | VEXER-CONN-ABS-01-001 | Deliver shared connector context/base classes so provider plug-ins can be activated via WebService/Worker. | -| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.WebService/TASKS.md | TODO | Team Vexer WebService | VEXER-WEB-01-001 | Scaffold minimal API host, DI, and `/vexer/status` endpoint integrating policy, storage, export, and attestation services. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Worker/TASKS.md | TODO | Team Vexer Worker | VEXER-WORKER-01-001 | Create Worker host with provider scheduling and logging to drive recurring pulls/reconciliation. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Formats.CSAF/TASKS.md | TODO | Team Vexer Formats | VEXER-FMT-CSAF-01-001 | Implement CSAF normalizer foundation translating provider documents into `VexClaim` entries. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Formats.CycloneDX/TASKS.md | TODO | Team Vexer Formats | VEXER-FMT-CYCLONE-01-001 | Implement CycloneDX VEX normalizer capturing `analysis` state and component references. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Formats.OpenVEX/TASKS.md | TODO | Team Vexer Formats | VEXER-FMT-OPENVEX-01-001 | Implement OpenVEX normalizer to ingest attestations into canonical claims with provenance. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.RedHat.CSAF/TASKS.md | TODO | Team Vexer Connectors – Red Hat | VEXER-CONN-RH-01-001 | Ship Red Hat CSAF provider metadata discovery enabling incremental pulls. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.Cisco.CSAF/TASKS.md | TODO | Team Vexer Connectors – Cisco | VEXER-CONN-CISCO-01-001 | Implement Cisco CSAF endpoint discovery/auth to unlock paginated pulls. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/TASKS.md | TODO | Team Vexer Connectors – SUSE | VEXER-CONN-SUSE-01-001 | Build Rancher VEX Hub discovery/subscription path with offline snapshot support. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.MSRC.CSAF/TASKS.md | TODO | Team Vexer Connectors – MSRC | VEXER-CONN-MS-01-001 | Deliver AAD onboarding/token cache for MSRC CSAF ingestion. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.Oracle.CSAF/TASKS.md | TODO | Team Vexer Connectors – Oracle | VEXER-CONN-ORACLE-01-001 | Implement Oracle CSAF catalogue discovery with CPU calendar awareness. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/TASKS.md | TODO | Team Vexer Connectors – Ubuntu | VEXER-CONN-UBUNTU-01-001 | Implement Ubuntu CSAF discovery and channel selection for USN ingestion. | +| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Export/TASKS.md | DONE (2025-10-17) | Team Vexer Export | VEXER-EXPORT-01-004 | Connect export engine to attestation client and persist Rekor metadata. | +| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Attestation/TASKS.md | DONE (2025-10-16) | Team Vexer Attestation | VEXER-ATTEST-01-001 | Implement in-toto predicate + DSSE builder providing envelopes for export attestation. | +| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Connectors.Abstractions/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors | VEXER-CONN-ABS-01-001 | Deliver shared connector context/base classes so provider plug-ins can be activated via WebService/Worker. | +| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.WebService/TASKS.md | DONE (2025-10-17) | Team Vexer WebService | VEXER-WEB-01-001 | Scaffold minimal API host, DI, and `/vexer/status` endpoint integrating policy, storage, export, and attestation services. | +| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Worker/TASKS.md | DONE (2025-10-17) | Team Vexer Worker | VEXER-WORKER-01-001 | Create Worker host with provider scheduling and logging to drive recurring pulls/reconciliation. | +| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Formats.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Formats | VEXER-FMT-CSAF-01-001 | Implement CSAF normalizer foundation translating provider documents into `VexClaim` entries. | +| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Formats.CycloneDX/TASKS.md | DONE (2025-10-17) | Team Vexer Formats | VEXER-FMT-CYCLONE-01-001 | Implement CycloneDX VEX normalizer capturing `analysis` state and component references. | +| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Formats.OpenVEX/TASKS.md | DONE (2025-10-17) | Team Vexer Formats | VEXER-FMT-OPENVEX-01-001 | Implement OpenVEX normalizer to ingest attestations into canonical claims with provenance. | +| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – Red Hat | VEXER-CONN-RH-01-001 | Ship Red Hat CSAF provider metadata discovery enabling incremental pulls. | +| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – Red Hat | VEXER-CONN-RH-01-002 | Fetch CSAF windows with ETag handling, resume tokens, quarantine on schema errors, and persist raw docs. | +| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – Red Hat | VEXER-CONN-RH-01-003 | Populate provider trust overrides (cosign issuer, identity regex) and provenance hints for policy evaluation/logging. | +| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – Red Hat | VEXER-CONN-RH-01-004 | Persist resume cursors (last updated timestamp/document hashes) in storage and reload during fetch to avoid duplicates. | +| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – Red Hat | VEXER-CONN-RH-01-005 | Register connector in Worker/WebService DI, add scheduled jobs, and document CLI triggers for Red Hat CSAF pulls. | +| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – Red Hat | VEXER-CONN-RH-01-006 | Add CSAF normalization parity fixtures ensuring RHSA-specific metadata is preserved. | +| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.Cisco.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – Cisco | VEXER-CONN-CISCO-01-001 | Implement Cisco CSAF endpoint discovery/auth to unlock paginated pulls. | +| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.Cisco.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – Cisco | VEXER-CONN-CISCO-01-002 | Implement Cisco CSAF paginated fetch loop with dedupe and raw persistence support. | +| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – SUSE | VEXER-CONN-SUSE-01-001 | Build Rancher VEX Hub discovery/subscription path with offline snapshot support. | +| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.MSRC.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – MSRC | VEXER-CONN-MS-01-001 | Deliver AAD onboarding/token cache for MSRC CSAF ingestion. | +| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.Oracle.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – Oracle | VEXER-CONN-ORACLE-01-001 | Implement Oracle CSAF catalogue discovery with CPU calendar awareness. | +| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – Ubuntu | VEXER-CONN-UBUNTU-01-001 | Implement Ubuntu CSAF discovery and channel selection for USN ingestion. | | Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.OCI.OpenVEX.Attest/TASKS.md | TODO | Team Vexer Connectors – OCI | VEXER-CONN-OCI-01-001 | Wire OCI discovery/auth to fetch OpenVEX attestations for configured images. | | Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI | VEXER-CLI-01-001 | Add `vexer` CLI verbs bridging to WebService with consistent auth and offline UX. | | Sprint 7 | Contextual Truth Foundations | src/StellaOps.Vexer.Core/TASKS.md | TODO | Team Vexer Core & Policy | VEXER-CORE-02-001 | Context signal schema prep – extend consensus models with severity/KEV/EPSS fields and update canonical serializers. | | Sprint 7 | Contextual Truth Foundations | src/StellaOps.Vexer.Policy/TASKS.md | TODO | Team Vexer Policy | VEXER-POLICY-02-001 | Scoring coefficients & weight ceilings – add α/β options, weight boosts, and validation guidance. | | Sprint 7 | Contextual Truth Foundations | src/StellaOps.Vexer.Storage.Mongo/TASKS.md | TODO | Team Vexer Storage | VEXER-STORAGE-02-001 | Statement events & scoring signals – create immutable VEX statement store plus consensus extensions with indexes/migrations. | | Sprint 7 | Contextual Truth Foundations | src/StellaOps.Vexer.WebService/TASKS.md | TODO | Team Vexer WebService | VEXER-WEB-01-004 | Resolve API & signed responses – expose `/vexer/resolve`, return signed consensus/score envelopes, document auth. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Vexer.Attestation/TASKS.md | TODO | Team Vexer Attestation | VEXER-ATTEST-01-002 | Rekor v2 client integration – ship transparency log client with retries and offline queue. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Vexer.Attestation/TASKS.md | DONE (2025-10-16) | Team Vexer Attestation | VEXER-ATTEST-01-002 | Rekor v2 client integration – ship transparency log client with retries and offline queue. | | Sprint 7 | Contextual Truth Foundations | src/StellaOps.Vexer.Worker/TASKS.md | TODO | Team Vexer Worker | VEXER-WORKER-01-004 | TTL refresh & stability damper – schedule re-resolve loops and guard against status flapping. | | Sprint 7 | Contextual Truth Foundations | src/StellaOps.Vexer.Export/TASKS.md | TODO | Team Vexer Export | VEXER-EXPORT-01-005 | Score & resolve envelope surfaces – include signed consensus/score artifacts in exports. | | Sprint 7 | Contextual Truth Foundations | src/StellaOps.Feedser.Core/TASKS.md | TODO | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-07-001 | Advisory event log & asOf queries – surface immutable statements and replay capability. | diff --git a/seed-data/cert-bund/README.md b/seed-data/cert-bund/README.md new file mode 100644 index 00000000..28c5c642 --- /dev/null +++ b/seed-data/cert-bund/README.md @@ -0,0 +1,52 @@ +# CERT-Bund Offline Kit Seed Data + +This directory stores **offline snapshots** for the CERT-Bund connector. +The artefacts mirror the public JSON search and export endpoints so +air‑gapped deployments can hydrate the connector without contacting the +portal. + +> ⚠️ **Distribution notice** – CERT-Bund advisories are published by BSI +> (Federal Office for Information Security, Germany). Review the portal +> terms of use before redistributing the snapshots. Always keep the JSON +> payloads and accompanying SHA-256 sums together. + +## Recommended layout + +``` +seed-data/cert-bund/ +├── search/ # paginated search JSON files +│   ├── certbund-search-page-00.json +│   └── … +├── export/ # yearly export JSON files +│   ├── certbund-export-2014.json +│   └── … +├── manifest/ +│   └── certbund-offline-manifest.json +└── certbund-offline-manifest.sha256 +``` + +Use `certbund-offline-manifest.json` to feed the Offline Kit build: every +entry contains `source`, `from`, `to`, `sha256`, `capturedAt`, and the +relative file path. The manifest is deterministic when regenerated with +the tooling described below. + +## Tooling + +Run the helper under `tools/` to capture fresh snapshots or regenerate +the manifest: + +``` +python tools/certbund_offline_snapshot.py --output seed-data/cert-bund +``` + +See the connector operations guide +(`docs/ops/feedser-certbund-operations.md`) for detailed usage, +including how to provide cookies/tokens when the portal requires manual +authentication. + +## Git hygiene + +- JSON payloads and checksums are **ignored by Git**. Generate them + locally when preparing an Offline Kit bundle. +- Commit documentation, scripts, and manifest templates only – never the + exported advisory data itself. diff --git a/src/StellaOps.Feedser.Source.CertBund/TASKS.md b/src/StellaOps.Feedser.Source.CertBund/TASKS.md index 0fb691ed..48c152f6 100644 --- a/src/StellaOps.Feedser.Source.CertBund/TASKS.md +++ b/src/StellaOps.Feedser.Source.CertBund/TASKS.md @@ -9,4 +9,4 @@ |FEEDCONN-CERTBUND-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-15)** – Added `CertBundDiagnostics` (meter `StellaOps.Feedser.Source.CertBund`) with fetch/parse/map counters + histograms, recorded coverage days, wired stage summary logs, and published the ops runbook (`docs/ops/feedser-certbund-operations.md`).| |FEEDCONN-CERTBUND-02-007 Feed history & locale assessment|BE-Conn-CERTBUND|Research|**DONE (2025-10-15)** – Measured RSS retention (~6 days/≈250 items), captured connector-driven backfill guidance in the runbook, and aligned locale guidance (preserve `language=de`, Docs glossary follow-up). **Next:** coordinate with Tools to land the state-seeding helper so scripted backfills replace manual Mongo tweaks.| |FEEDCONN-CERTBUND-02-008 Session bootstrap & cookie strategy|BE-Conn-CERTBUND|Source.Common|**DONE (2025-10-14)** – Feed client primes the portal session (cookie container via `SocketsHttpHandler`), shares cookies across detail requests, and documents bootstrap behaviour in options (`PortalBootstrapUri`).| -|FEEDCONN-CERTBUND-02-009 Offline Kit export packaging|BE-Conn-CERTBUND, Docs|Offline Kit|**TODO** – Capture JSON search/export snapshots (per-year splits), generate manifest fields (`source`,`from`,`to`,`sha256`,`capturedAt`), and update Offline Kit docs so air-gapped deployments can seed historical CERT-Bund advisories without live fetching. **Remark:** follow the interim workflow documented in `docs/ops/feedser-certbund-operations.md` §3.3 until the packaged artefacts ship.| +|FEEDCONN-CERTBUND-02-009 Offline Kit export packaging|BE-Conn-CERTBUND, Docs|Offline Kit|**DONE (2025-10-17)** – Added `tools/certbund_offline_snapshot.py` to capture search/export JSON, emit deterministic manifests + SHA files, and refreshed docs (`docs/ops/feedser-certbund-operations.md`, `docs/24_OFFLINE_KIT.md`) with offline-kit instructions and manifest layout guidance. Seed data README/ignore rules cover local snapshot hygiene.| diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/AGENTS.md b/src/StellaOps.Vexer.Connectors.Abstractions/AGENTS.md index 6d4e8e97..dadb0d0d 100644 --- a/src/StellaOps.Vexer.Connectors.Abstractions/AGENTS.md +++ b/src/StellaOps.Vexer.Connectors.Abstractions/AGENTS.md @@ -5,12 +5,12 @@ Defines shared connector infrastructure for Vexer, including base contexts, resu - `IVexConnector` context implementation, raw store helpers, verification hooks, and telemetry utilities. - Configuration primitives (YAML parsing, secrets handling guidelines) and options validation. - Connector lifecycle helpers for retries, paging, `.well-known` discovery, and resume markers. -- Documentation for connector packaging, plugin manifest metadata, and DI registration. +- Documentation for connector packaging, plugin manifest metadata, and DI registration (see `docs/dev/30_VEXER_CONNECTOR_GUIDE.md` and `docs/dev/templates/vexer-connector/`). ## Participants - All Vexer connector projects reference this module to obtain base classes and context services. - WebService/Worker instantiate connectors via plugin loader leveraging abstractions defined here. ## Interfaces & contracts -- Connector context, result, and telemetry interfaces; `ConnectorDescriptor`, `ConnectorOptions`, authentication helpers. +- Connector context, result, and telemetry interfaces; `VexConnectorDescriptor`, `VexConnectorBase`, options binder/validators, authentication helpers. - Utility classes for HTTP clients, throttling, and deterministic logging. ## In/Out of scope In: shared abstractions, helper utilities, configuration binding, documentation for connector authors. diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/IVexConnectorOptionsValidator.cs b/src/StellaOps.Vexer.Connectors.Abstractions/IVexConnectorOptionsValidator.cs new file mode 100644 index 00000000..007004d7 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Abstractions/IVexConnectorOptionsValidator.cs @@ -0,0 +1,12 @@ +using System.Collections.Generic; + +namespace StellaOps.Vexer.Connectors.Abstractions; + +/// +/// Custom validator hook executed after connector options are bound. +/// +/// Connector-specific options type. +public interface IVexConnectorOptionsValidator +{ + void Validate(VexConnectorDescriptor descriptor, TOptions options, IList errors); +} diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/StellaOps.Vexer.Connectors.Abstractions.csproj b/src/StellaOps.Vexer.Connectors.Abstractions/StellaOps.Vexer.Connectors.Abstractions.csproj new file mode 100644 index 00000000..63f633c3 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Abstractions/StellaOps.Vexer.Connectors.Abstractions.csproj @@ -0,0 +1,17 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/TASKS.md b/src/StellaOps.Vexer.Connectors.Abstractions/TASKS.md index 408ff364..696fe975 100644 --- a/src/StellaOps.Vexer.Connectors.Abstractions/TASKS.md +++ b/src/StellaOps.Vexer.Connectors.Abstractions/TASKS.md @@ -2,6 +2,6 @@ If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and # TASKS | Task | Owner(s) | Depends on | Notes | |---|---|---|---| -|VEXER-CONN-ABS-01-001 – Connector context & base classes|Team Vexer Connectors|VEXER-CORE-01-003|TODO – Implement `VexConnectorContext`, result types, helper base classes, and deterministic logging helpers for connectors.| -|VEXER-CONN-ABS-01-002 – YAML options & validation|Team Vexer Connectors|VEXER-CONN-ABS-01-001|TODO – Provide strongly-typed options binding/validation for connector YAML definitions with offline-safe defaults.| -|VEXER-CONN-ABS-01-003 – Plugin packaging & docs|Team Vexer Connectors|VEXER-CONN-ABS-01-001|TODO – Document connector packaging (NuGet manifest, plugin loader metadata) and supply reference templates for downstream connector modules.| +|VEXER-CONN-ABS-01-001 – Connector context & base classes|Team Vexer Connectors|VEXER-CORE-01-003|**DONE (2025-10-17)** – Added `StellaOps.Vexer.Connectors.Abstractions` project with `VexConnectorBase`, deterministic logging scopes, metadata builder helpers, and connector descriptors; docs updated to highlight the shared abstractions.| +|VEXER-CONN-ABS-01-002 – YAML options & validation|Team Vexer Connectors|VEXER-CONN-ABS-01-001|**DONE (2025-10-17)** – Delivered `VexConnectorOptionsBinder` + binder options/validators, environment-variable expansion, data-annotation checks, and custom validation hooks with documentation updates covering the workflow.| +|VEXER-CONN-ABS-01-003 – Plugin packaging & docs|Team Vexer Connectors|VEXER-CONN-ABS-01-001|**DONE (2025-10-17)** – Authored `docs/dev/30_VEXER_CONNECTOR_GUIDE.md`, added quick-start template under `docs/dev/templates/vexer-connector/`, and updated module docs to reference the packaging workflow.| diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorBase.cs b/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorBase.cs new file mode 100644 index 00000000..ba312419 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorBase.cs @@ -0,0 +1,99 @@ +using System.Collections.Immutable; +using System.Security.Cryptography; +using Microsoft.Extensions.Logging; +using StellaOps.Vexer.Core; + +namespace StellaOps.Vexer.Connectors.Abstractions; + +/// +/// Convenience base class for implementing . +/// +public abstract class VexConnectorBase : IVexConnector +{ + protected VexConnectorBase(VexConnectorDescriptor descriptor, ILogger logger, TimeProvider? timeProvider = null) + { + Descriptor = descriptor ?? throw new ArgumentNullException(nameof(descriptor)); + Logger = logger ?? throw new ArgumentNullException(nameof(logger)); + TimeProvider = timeProvider ?? TimeProvider.System; + } + + /// + public string Id => Descriptor.Id; + + /// + public VexProviderKind Kind => Descriptor.Kind; + + protected VexConnectorDescriptor Descriptor { get; } + + protected ILogger Logger { get; } + + protected TimeProvider TimeProvider { get; } + + protected DateTimeOffset UtcNow() => TimeProvider.GetUtcNow(); + + protected VexRawDocument CreateRawDocument( + VexDocumentFormat format, + Uri sourceUri, + ReadOnlyMemory content, + ImmutableDictionary? metadata = null) + { + if (sourceUri is null) + { + throw new ArgumentNullException(nameof(sourceUri)); + } + + var digest = ComputeSha256(content.Span); + var captured = TimeProvider.GetUtcNow(); + return new VexRawDocument( + Descriptor.Id, + format, + sourceUri, + captured, + digest, + content, + metadata ?? ImmutableDictionary.Empty); + } + + protected IDisposable BeginConnectorScope(string operation, IReadOnlyDictionary? metadata = null) + => VexConnectorLogScope.Begin(Logger, Descriptor, operation, metadata); + + protected void LogConnectorEvent(LogLevel level, string eventName, string message, IReadOnlyDictionary? metadata = null, Exception? exception = null) + { + using var scope = BeginConnectorScope(eventName, metadata); + if (exception is null) + { + Logger.Log(level, "{Message}", message); + } + else + { + Logger.Log(level, exception, "{Message}", message); + } + } + + protected ImmutableDictionary BuildMetadata(Action configure) + { + ArgumentNullException.ThrowIfNull(configure); + var builder = new VexConnectorMetadataBuilder(); + configure(builder); + return builder.Build(); + } + + private static string ComputeSha256(ReadOnlySpan content) + { + Span buffer = stackalloc byte[32]; + if (SHA256.TryHashData(content, buffer, out _)) + { + return "sha256:" + Convert.ToHexString(buffer).ToLowerInvariant(); + } + + using var sha = SHA256.Create(); + var hash = sha.ComputeHash(content.ToArray()); + return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant(); + } + + public abstract ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken); + + public abstract IAsyncEnumerable FetchAsync(VexConnectorContext context, CancellationToken cancellationToken); + + public abstract ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorDescriptor.cs b/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorDescriptor.cs new file mode 100644 index 00000000..4e7eb211 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorDescriptor.cs @@ -0,0 +1,54 @@ +using System.Collections.Immutable; +using StellaOps.Vexer.Core; + +namespace StellaOps.Vexer.Connectors.Abstractions; + +/// +/// Static descriptor for a Vexer connector plug-in. +/// +public sealed record VexConnectorDescriptor +{ + public VexConnectorDescriptor(string id, VexProviderKind kind, string displayName) + { + if (string.IsNullOrWhiteSpace(id)) + { + throw new ArgumentException("Connector id must be provided.", nameof(id)); + } + + Id = id; + Kind = kind; + DisplayName = string.IsNullOrWhiteSpace(displayName) ? id : displayName; + } + + /// + /// Stable connector identifier (matches provider id). + /// + public string Id { get; } + + /// + /// Provider kind served by the connector. + /// + public VexProviderKind Kind { get; } + + /// + /// Human friendly name used in logs/diagnostics. + /// + public string DisplayName { get; } + + /// + /// Optional friendly description. + /// + public string? Description { get; init; } + + /// + /// Document formats the connector is expected to emit. + /// + public ImmutableArray SupportedFormats { get; init; } = ImmutableArray.Empty; + + /// + /// Optional tags surfaced in diagnostics (e.g. "beta", "offline"). + /// + public ImmutableArray Tags { get; init; } = ImmutableArray.Empty; + + public override string ToString() => $"{Id} ({Kind})"; +} diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorLogScope.cs b/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorLogScope.cs new file mode 100644 index 00000000..5cc8da82 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorLogScope.cs @@ -0,0 +1,50 @@ +using System.Linq; +using Microsoft.Extensions.Logging; +using StellaOps.Vexer.Core; + +namespace StellaOps.Vexer.Connectors.Abstractions; + +/// +/// Helper to establish deterministic logging scopes for connector operations. +/// +public static class VexConnectorLogScope +{ + public static IDisposable Begin(ILogger logger, VexConnectorDescriptor descriptor, string operation, IReadOnlyDictionary? metadata = null) + { + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(descriptor); + ArgumentException.ThrowIfNullOrEmpty(operation); + + var scopeValues = new List> + { + new("vex.connector.id", descriptor.Id), + new("vex.connector.kind", descriptor.Kind.ToString()), + new("vex.connector.operation", operation), + }; + + if (!string.Equals(descriptor.DisplayName, descriptor.Id, StringComparison.Ordinal)) + { + scopeValues.Add(new KeyValuePair("vex.connector.displayName", descriptor.DisplayName)); + } + + if (!string.IsNullOrWhiteSpace(descriptor.Description)) + { + scopeValues.Add(new KeyValuePair("vex.connector.description", descriptor.Description)); + } + + if (!descriptor.Tags.IsDefaultOrEmpty) + { + scopeValues.Add(new KeyValuePair("vex.connector.tags", string.Join(",", descriptor.Tags))); + } + + if (metadata is not null) + { + foreach (var kvp in metadata.OrderBy(static pair => pair.Key, StringComparer.Ordinal)) + { + scopeValues.Add(new KeyValuePair($"vex.{kvp.Key}", kvp.Value)); + } + } + + return logger.BeginScope(scopeValues)!; + } +} diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorMetadataBuilder.cs b/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorMetadataBuilder.cs new file mode 100644 index 00000000..8404ab94 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorMetadataBuilder.cs @@ -0,0 +1,37 @@ +using System.Collections.Immutable; + +namespace StellaOps.Vexer.Connectors.Abstractions; + +/// +/// Builds deterministic metadata dictionaries for raw documents and logging scopes. +/// +public sealed class VexConnectorMetadataBuilder +{ + private readonly SortedDictionary _values = new(StringComparer.Ordinal); + + public VexConnectorMetadataBuilder Add(string key, string? value) + { + if (!string.IsNullOrWhiteSpace(key) && !string.IsNullOrWhiteSpace(value)) + { + _values[key] = value!; + } + + return this; + } + + public VexConnectorMetadataBuilder Add(string key, DateTimeOffset value) + => Add(key, value.ToUniversalTime().ToString("O")); + + public VexConnectorMetadataBuilder AddRange(IEnumerable> items) + { + foreach (var item in items) + { + Add(item.Key, item.Value); + } + + return this; + } + + public ImmutableDictionary Build() + => _values.ToImmutableDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.Ordinal); +} diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorOptionsBinder.cs b/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorOptionsBinder.cs new file mode 100644 index 00000000..d307fbca --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorOptionsBinder.cs @@ -0,0 +1,157 @@ +using System.Collections.Immutable; +using System.ComponentModel.DataAnnotations; +using System.Linq; +using Microsoft.Extensions.Configuration; +using StellaOps.Vexer.Core; + +namespace StellaOps.Vexer.Connectors.Abstractions; + +/// +/// Provides strongly typed binding and validation for connector options. +/// +public static class VexConnectorOptionsBinder +{ + public static TOptions Bind( + VexConnectorDescriptor descriptor, + VexConnectorSettings settings, + VexConnectorOptionsBinderOptions? options = null, + IEnumerable>? validators = null) + where TOptions : class, new() + { + ArgumentNullException.ThrowIfNull(descriptor); + ArgumentNullException.ThrowIfNull(settings); + + var binderSettings = options ?? new VexConnectorOptionsBinderOptions(); + var transformed = TransformValues(settings, binderSettings); + + var configuration = BuildConfiguration(transformed); + + var result = new TOptions(); + var errors = new List(); + + try + { + configuration.Bind( + result, + binderOptions => binderOptions.ErrorOnUnknownConfiguration = !binderSettings.AllowUnknownKeys); + } + catch (InvalidOperationException ex) when (!binderSettings.AllowUnknownKeys) + { + errors.Add(ex.Message); + } + + binderSettings.PostConfigure?.Invoke(result); + + if (binderSettings.ValidateDataAnnotations) + { + ValidateDataAnnotations(result, errors); + } + + if (validators is not null) + { + foreach (var validator in validators) + { + validator?.Validate(descriptor, result, errors); + } + } + + if (errors.Count > 0) + { + throw new VexConnectorOptionsValidationException(descriptor.Id, errors); + } + + return result; + } + + private static ImmutableDictionary TransformValues( + VexConnectorSettings settings, + VexConnectorOptionsBinderOptions binderOptions) + { + var builder = ImmutableDictionary.CreateBuilder(StringComparer.OrdinalIgnoreCase); + foreach (var kvp in settings.Values) + { + var value = kvp.Value; + if (binderOptions.TrimWhitespace && value is not null) + { + value = value.Trim(); + } + + if (binderOptions.TreatEmptyAsNull && string.IsNullOrEmpty(value)) + { + value = null; + } + + if (value is not null && binderOptions.ExpandEnvironmentVariables) + { + value = Environment.ExpandEnvironmentVariables(value); + } + + if (binderOptions.ValueTransformer is not null) + { + value = binderOptions.ValueTransformer.Invoke(kvp.Key, value); + } + + builder[kvp.Key] = value; + } + + return builder.ToImmutable(); + } + + private static IConfiguration BuildConfiguration(ImmutableDictionary values) + { + var sources = new List>(); + foreach (var kvp in values) + { + if (kvp.Value is not null) + { + sources.Add(new KeyValuePair(kvp.Key, kvp.Value)); + } + } + + var configurationBuilder = new ConfigurationBuilder(); + configurationBuilder.Add(new DictionaryConfigurationSource(sources)); + return configurationBuilder.Build(); + } + + private static void ValidateDataAnnotations(TOptions options, IList errors) + { + var validationResults = new List(); + var validationContext = new ValidationContext(options!); + if (!Validator.TryValidateObject(options!, validationContext, validationResults, validateAllProperties: true)) + { + foreach (var validationResult in validationResults) + { + if (!string.IsNullOrWhiteSpace(validationResult.ErrorMessage)) + { + errors.Add(validationResult.ErrorMessage); + } + } + } + } + + private sealed class DictionaryConfigurationSource : IConfigurationSource + { + private readonly IReadOnlyList> _data; + + public DictionaryConfigurationSource(IEnumerable> data) + { + _data = data?.ToList() ?? new List>(); + } + + public IConfigurationProvider Build(IConfigurationBuilder builder) => new DictionaryConfigurationProvider(_data); + } + + private sealed class DictionaryConfigurationProvider : ConfigurationProvider + { + public DictionaryConfigurationProvider(IEnumerable> data) + { + foreach (var pair in data) + { + if (pair.Value is not null) + { + Data[pair.Key] = pair.Value; + } + } + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorOptionsBinderOptions.cs b/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorOptionsBinderOptions.cs new file mode 100644 index 00000000..c4710999 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorOptionsBinderOptions.cs @@ -0,0 +1,45 @@ +namespace StellaOps.Vexer.Connectors.Abstractions; + +/// +/// Customisation options for connector options binding. +/// +public sealed class VexConnectorOptionsBinderOptions +{ + /// + /// Indicates whether environment variables should be expanded in option values. + /// Defaults to true. + /// + public bool ExpandEnvironmentVariables { get; set; } = true; + + /// + /// When true the binder trims whitespace around option values. + /// + public bool TrimWhitespace { get; set; } = true; + + /// + /// Converts empty strings to null before binding. Default: true. + /// + public bool TreatEmptyAsNull { get; set; } = true; + + /// + /// When false, binding fails if unknown configuration keys are provided. + /// Default: true (permitting unknown keys). + /// + public bool AllowUnknownKeys { get; set; } = true; + + /// + /// Enables validation after binding. + /// Default: true. + /// + public bool ValidateDataAnnotations { get; set; } = true; + + /// + /// Optional post-configuration callback executed after binding. + /// + public Action? PostConfigure { get; set; } + + /// + /// Optional hook to transform raw configuration values before binding. + /// + public Func? ValueTransformer { get; set; } +} diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorOptionsValidationException.cs b/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorOptionsValidationException.cs new file mode 100644 index 00000000..4bd9b48c --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorOptionsValidationException.cs @@ -0,0 +1,36 @@ +using System.Collections.Immutable; + +namespace StellaOps.Vexer.Connectors.Abstractions; + +public sealed class VexConnectorOptionsValidationException : Exception +{ + public VexConnectorOptionsValidationException( + string connectorId, + IEnumerable errors) + : base(BuildMessage(connectorId, errors)) + { + ConnectorId = connectorId; + Errors = errors?.ToImmutableArray() ?? ImmutableArray.Empty; + } + + public string ConnectorId { get; } + + public ImmutableArray Errors { get; } + + private static string BuildMessage(string connectorId, IEnumerable errors) + { + var builder = new System.Text.StringBuilder(); + builder.Append("Connector options validation failed for '"); + builder.Append(connectorId); + builder.Append("'."); + + var list = errors?.ToImmutableArray() ?? ImmutableArray.Empty; + if (!list.IsDefaultOrEmpty) + { + builder.Append(" Errors: "); + builder.Append(string.Join("; ", list)); + } + + return builder.ToString(); + } +} diff --git a/src/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests/Connectors/CiscoCsafConnectorTests.cs b/src/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests/Connectors/CiscoCsafConnectorTests.cs new file mode 100644 index 00000000..792f3326 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests/Connectors/CiscoCsafConnectorTests.cs @@ -0,0 +1,214 @@ +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Vexer.Connectors.Abstractions; +using StellaOps.Vexer.Connectors.Cisco.CSAF; +using StellaOps.Vexer.Connectors.Cisco.CSAF.Configuration; +using StellaOps.Vexer.Connectors.Cisco.CSAF.Metadata; +using StellaOps.Vexer.Core; +using StellaOps.Vexer.Storage.Mongo; +using System.Collections.Immutable; +using System.IO.Abstractions.TestingHelpers; +using Xunit; +using System.Threading; + +namespace StellaOps.Vexer.Connectors.Cisco.CSAF.Tests.Connectors; + +public sealed class CiscoCsafConnectorTests +{ + [Fact] + public async Task FetchAsync_NewAdvisory_StoresDocumentAndUpdatesState() + { + var responses = new Dictionary> + { + [new Uri("https://api.cisco.test/.well-known/csaf/provider-metadata.json")] = QueueResponses(""" + { + "metadata": { + "publisher": { + "name": "Cisco", + "category": "vendor", + "contact_details": { "id": "vexer:cisco" } + } + }, + "distributions": { + "directories": [ "https://api.cisco.test/csaf/" ] + } + } + """), + [new Uri("https://api.cisco.test/csaf/index.json")] = QueueResponses(""" + { + "advisories": [ + { + "id": "cisco-sa-2025", + "url": "https://api.cisco.test/csaf/cisco-sa-2025.json", + "published": "2025-10-01T00:00:00Z", + "lastModified": "2025-10-02T00:00:00Z", + "sha256": "cafebabe" + } + ] + } + """), + [new Uri("https://api.cisco.test/csaf/cisco-sa-2025.json")] = QueueResponses("{ \"document\": \"payload\" }") + }; + + var handler = new RoutingHttpMessageHandler(responses); + var httpClient = new HttpClient(handler); + var factory = new SingleHttpClientFactory(httpClient); + var metadataLoader = new CiscoProviderMetadataLoader( + factory, + new MemoryCache(new MemoryCacheOptions()), + Options.Create(new CiscoConnectorOptions + { + MetadataUri = "https://api.cisco.test/.well-known/csaf/provider-metadata.json", + PersistOfflineSnapshot = false, + }), + NullLogger.Instance, + new MockFileSystem()); + + var stateRepository = new InMemoryConnectorStateRepository(); + var connector = new CiscoCsafConnector( + metadataLoader, + factory, + stateRepository, + new[] { new CiscoConnectorOptionsValidator() }, + NullLogger.Instance, + TimeProvider.System); + + var settings = new VexConnectorSettings(ImmutableDictionary.Empty); + await connector.ValidateAsync(settings, CancellationToken.None); + + var sink = new InMemoryRawSink(); + var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider()); + + var documents = new List(); + await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(doc); + } + + documents.Should().HaveCount(1); + sink.Documents.Should().HaveCount(1); + stateRepository.CurrentState.Should().NotBeNull(); + stateRepository.CurrentState!.DocumentDigests.Should().HaveCount(1); + + // second run should not refetch documents + sink.Documents.Clear(); + documents.Clear(); + + await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(doc); + } + + documents.Should().BeEmpty(); + sink.Documents.Should().BeEmpty(); + } + + private static Queue QueueResponses(string payload) + => new(new[] + { + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(payload, Encoding.UTF8, "application/json"), + } + }); + + private sealed class RoutingHttpMessageHandler : HttpMessageHandler + { + private readonly Dictionary> _responses; + + public RoutingHttpMessageHandler(Dictionary> responses) + { + _responses = responses; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (request.RequestUri is not null && _responses.TryGetValue(request.RequestUri, out var queue) && queue.Count > 0) + { + var response = queue.Peek(); + return Task.FromResult(response.Clone()); + } + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) + { + Content = new StringContent($"No response configured for {request.RequestUri}"), + }); + } + } + + private sealed class SingleHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository + { + public VexConnectorState? CurrentState { get; private set; } + + public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken) + => ValueTask.FromResult(CurrentState); + + public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken) + { + CurrentState = state; + return ValueTask.CompletedTask; + } + } + + private sealed class InMemoryRawSink : IVexRawDocumentSink + { + public List Documents { get; } = new(); + + public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) + { + Documents.Add(document); + return ValueTask.CompletedTask; + } + } + + private sealed class NoopSignatureVerifier : IVexSignatureVerifier + { + public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(null); + } + + private sealed class NoopNormalizerRouter : IVexNormalizerRouter + { + public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); + } +} + +internal static class HttpResponseMessageExtensions +{ + public static HttpResponseMessage Clone(this HttpResponseMessage response) + { + var clone = new HttpResponseMessage(response.StatusCode); + foreach (var header in response.Headers) + { + clone.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + + if (response.Content is not null) + { + var payload = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); + clone.Content = new StringContent(payload, Encoding.UTF8, response.Content.Headers.ContentType?.MediaType); + } + + return clone; + } +} diff --git a/src/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests/Metadata/CiscoProviderMetadataLoaderTests.cs b/src/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests/Metadata/CiscoProviderMetadataLoaderTests.cs new file mode 100644 index 00000000..59b0031d --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests/Metadata/CiscoProviderMetadataLoaderTests.cs @@ -0,0 +1,149 @@ +using System.Net; +using System.Net.Http; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Vexer.Connectors.Cisco.CSAF.Configuration; +using StellaOps.Vexer.Connectors.Cisco.CSAF.Metadata; +using StellaOps.Vexer.Core; +using System.IO.Abstractions.TestingHelpers; + +namespace StellaOps.Vexer.Connectors.Cisco.CSAF.Tests.Metadata; + +public sealed class CiscoProviderMetadataLoaderTests +{ + [Fact] + public async Task LoadAsync_FetchesFromNetworkWithBearerToken() + { + var payload = """ + { + "metadata": { + "publisher": { + "name": "Cisco CSAF", + "category": "vendor", + "contact_details": { + "id": "vexer:cisco" + } + } + }, + "distributions": { + "directories": [ + "https://api.security.cisco.com/csaf/v2/advisories/" + ] + }, + "discovery": { + "well_known": "https://api.security.cisco.com/.well-known/csaf/provider-metadata.json", + "rolie": "https://api.security.cisco.com/csaf/rolie/feed" + }, + "trust": { + "weight": 0.9, + "cosign": { + "issuer": "https://oidc.security.cisco.com", + "identity_pattern": "spiffe://cisco/*" + }, + "pgp_fingerprints": [ "1234ABCD" ] + } + } + """; + + HttpRequestMessage? capturedRequest = null; + var handler = new FakeHttpMessageHandler(request => + { + capturedRequest = request; + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(payload, Encoding.UTF8, "application/json"), + Headers = { ETag = new System.Net.Http.Headers.EntityTagHeaderValue("\"etag1\"") } + }; + }); + + var httpClient = new HttpClient(handler); + var factory = new SingleHttpClientFactory(httpClient); + var cache = new MemoryCache(new MemoryCacheOptions()); + var options = Options.Create(new CiscoConnectorOptions + { + ApiToken = "token-123", + MetadataUri = "https://api.security.cisco.com/.well-known/csaf/provider-metadata.json", + }); + var fileSystem = new MockFileSystem(); + var loader = new CiscoProviderMetadataLoader(factory, cache, options, NullLogger.Instance, fileSystem); + + var result = await loader.LoadAsync(CancellationToken.None); + + result.Provider.Id.Should().Be("vexer:cisco"); + result.Provider.BaseUris.Should().ContainSingle(uri => uri.ToString() == "https://api.security.cisco.com/csaf/v2/advisories/"); + result.Provider.Discovery.RolIeService.Should().Be(new Uri("https://api.security.cisco.com/csaf/rolie/feed")); + result.ServedFromCache.Should().BeFalse(); + capturedRequest.Should().NotBeNull(); + capturedRequest!.Headers.Authorization.Should().NotBeNull(); + capturedRequest.Headers.Authorization!.Parameter.Should().Be("token-123"); + } + + [Fact] + public async Task LoadAsync_FallsBackToOfflineSnapshot() + { + var payload = """ + { + "metadata": { + "publisher": { + "name": "Cisco CSAF", + "category": "vendor", + "contact_details": { + "id": "vexer:cisco" + } + } + } + } + """; + + var handler = new FakeHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.InternalServerError)); + var httpClient = new HttpClient(handler); + var factory = new SingleHttpClientFactory(httpClient); + var cache = new MemoryCache(new MemoryCacheOptions()); + var options = Options.Create(new CiscoConnectorOptions + { + MetadataUri = "https://api.security.cisco.com/.well-known/csaf/provider-metadata.json", + PreferOfflineSnapshot = true, + OfflineSnapshotPath = "/snapshots/cisco.json", + }); + var fileSystem = new MockFileSystem(new Dictionary + { + ["/snapshots/cisco.json"] = new MockFileData(payload), + }); + var loader = new CiscoProviderMetadataLoader(factory, cache, options, NullLogger.Instance, fileSystem); + + var result = await loader.LoadAsync(CancellationToken.None); + + result.FromOfflineSnapshot.Should().BeTrue(); + result.Provider.Id.Should().Be("vexer:cisco"); + } + + private sealed class SingleHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class FakeHttpMessageHandler : HttpMessageHandler + { + private readonly Func _responder; + + public FakeHttpMessageHandler(Func responder) + { + _responder = responder; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + return Task.FromResult(_responder(request)); + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests.csproj b/src/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests.csproj new file mode 100644 index 00000000..1e11d778 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests.csproj @@ -0,0 +1,16 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + diff --git a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/CiscoCsafConnector.cs b/src/StellaOps.Vexer.Connectors.Cisco.CSAF/CiscoCsafConnector.cs new file mode 100644 index 00000000..a9829c3c --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Cisco.CSAF/CiscoCsafConnector.cs @@ -0,0 +1,247 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using StellaOps.Vexer.Connectors.Abstractions; +using StellaOps.Vexer.Connectors.Cisco.CSAF.Configuration; +using StellaOps.Vexer.Connectors.Cisco.CSAF.Metadata; +using StellaOps.Vexer.Core; +using StellaOps.Vexer.Storage.Mongo; + +namespace StellaOps.Vexer.Connectors.Cisco.CSAF; + +public sealed class CiscoCsafConnector : VexConnectorBase +{ + private static readonly VexConnectorDescriptor DescriptorInstance = new( + id: "vexer:cisco", + kind: VexProviderKind.Vendor, + displayName: "Cisco CSAF") + { + Tags = ImmutableArray.Create("cisco", "csaf"), + }; + + private readonly CiscoProviderMetadataLoader _metadataLoader; + private readonly IHttpClientFactory _httpClientFactory; + private readonly IVexConnectorStateRepository _stateRepository; + private readonly IEnumerable> _validators; + private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web); + + private CiscoConnectorOptions? _options; + private CiscoProviderMetadataResult? _providerMetadata; + + public CiscoCsafConnector( + CiscoProviderMetadataLoader metadataLoader, + IHttpClientFactory httpClientFactory, + IVexConnectorStateRepository stateRepository, + IEnumerable>? validators, + ILogger logger, + TimeProvider timeProvider) + : base(DescriptorInstance, logger, timeProvider) + { + _metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader)); + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _validators = validators ?? Array.Empty>(); + } + + public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) + { + _options = VexConnectorOptionsBinder.Bind( + Descriptor, + settings, + validators: _validators); + + _providerMetadata = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false); + LogConnectorEvent(LogLevel.Information, "validate", "Cisco CSAF metadata loaded.", new Dictionary + { + ["baseUriCount"] = _providerMetadata.Provider.BaseUris.Length, + ["fromOffline"] = _providerMetadata.FromOfflineSnapshot, + }); + } + + public override async IAsyncEnumerable FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + if (_options is null) + { + throw new InvalidOperationException("Connector must be validated before fetch operations."); + } + + if (_providerMetadata is null) + { + _providerMetadata = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false); + } + + var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false); + var knownDigests = state?.DocumentDigests ?? ImmutableArray.Empty; + var digestSet = new HashSet(knownDigests, StringComparer.OrdinalIgnoreCase); + var digestList = new List(knownDigests); + var since = context.Since ?? state?.LastUpdated ?? DateTimeOffset.MinValue; + var latestTimestamp = state?.LastUpdated ?? since; + var stateChanged = false; + + var client = _httpClientFactory.CreateClient(CiscoConnectorOptions.HttpClientName); + foreach (var directory in _providerMetadata.Provider.BaseUris) + { + await foreach (var advisory in EnumerateCatalogAsync(client, directory, cancellationToken).ConfigureAwait(false)) + { + var published = advisory.LastModified ?? advisory.Published ?? DateTimeOffset.MinValue; + if (published <= since) + { + continue; + } + + using var contentResponse = await client.GetAsync(advisory.DocumentUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + contentResponse.EnsureSuccessStatusCode(); + var payload = await contentResponse.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + + var rawDocument = CreateRawDocument( + VexDocumentFormat.Csaf, + advisory.DocumentUri, + payload, + BuildMetadata(builder => builder + .Add("cisco.csaf.advisoryId", advisory.Id) + .Add("cisco.csaf.revision", advisory.Revision) + .Add("cisco.csaf.published", advisory.Published?.ToString("O")) + .Add("cisco.csaf.modified", advisory.LastModified?.ToString("O")) + .Add("cisco.csaf.sha256", advisory.Sha256))); + + if (!digestSet.Add(rawDocument.Digest)) + { + continue; + } + + await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false); + digestList.Add(rawDocument.Digest); + stateChanged = true; + if (published > latestTimestamp) + { + latestTimestamp = published; + } + + yield return rawDocument; + } + } + + if (stateChanged) + { + var newState = new VexConnectorState( + Descriptor.Id, + latestTimestamp == DateTimeOffset.MinValue ? state?.LastUpdated : latestTimestamp, + digestList.ToImmutableArray()); + await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false); + } + } + + public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => throw new NotSupportedException("CiscoCsafConnector relies on CSAF normalizers for document processing."); + + private async IAsyncEnumerable EnumerateCatalogAsync(HttpClient client, Uri directory, [EnumeratorCancellation] CancellationToken cancellationToken) + { + var nextUri = BuildIndexUri(directory, null); + while (nextUri is not null) + { + using var response = await client.GetAsync(nextUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var page = JsonSerializer.Deserialize(json, _serializerOptions); + if (page?.Advisories is null) + { + yield break; + } + + foreach (var advisory in page.Advisories) + { + if (string.IsNullOrWhiteSpace(advisory.Url)) + { + continue; + } + + if (!Uri.TryCreate(advisory.Url, UriKind.RelativeOrAbsolute, out var documentUri)) + { + continue; + } + + if (!documentUri.IsAbsoluteUri) + { + documentUri = new Uri(directory, documentUri); + } + + yield return new CiscoAdvisoryEntry( + advisory.Id ?? documentUri.Segments.LastOrDefault()?.Trim('/') ?? documentUri.ToString(), + documentUri, + advisory.Revision, + advisory.Published, + advisory.LastModified, + advisory.Sha256); + } + + nextUri = ResolveNextUri(directory, page.Next); + } + } + + private static Uri BuildIndexUri(Uri directory, string? relative) + { + if (string.IsNullOrWhiteSpace(relative)) + { + var baseText = directory.ToString(); + if (!baseText.EndsWith('/')) + { + baseText += "/"; + } + + return new Uri(new Uri(baseText, UriKind.Absolute), "index.json"); + } + + if (Uri.TryCreate(relative, UriKind.Absolute, out var absolute)) + { + return absolute; + } + + var baseTextRelative = directory.ToString(); + if (!baseTextRelative.EndsWith('/')) + { + baseTextRelative += "/"; + } + + return new Uri(new Uri(baseTextRelative, UriKind.Absolute), relative); + } + + private static Uri? ResolveNextUri(Uri directory, string? next) + { + if (string.IsNullOrWhiteSpace(next)) + { + return null; + } + + return BuildIndexUri(directory, next); + } + + private sealed record CiscoAdvisoryIndex + { + public List? Advisories { get; init; } + public string? Next { get; init; } + } + + private sealed record CiscoAdvisory + { + public string? Id { get; init; } + public string? Url { get; init; } + public string? Revision { get; init; } + public DateTimeOffset? Published { get; init; } + public DateTimeOffset? LastModified { get; init; } + public string? Sha256 { get; init; } + } + + private sealed record CiscoAdvisoryEntry( + string Id, + Uri DocumentUri, + string? Revision, + DateTimeOffset? Published, + DateTimeOffset? LastModified, + string? Sha256); +} diff --git a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptions.cs b/src/StellaOps.Vexer.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptions.cs new file mode 100644 index 00000000..20ee0ce5 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptions.cs @@ -0,0 +1,58 @@ +using System.ComponentModel.DataAnnotations; + +namespace StellaOps.Vexer.Connectors.Cisco.CSAF.Configuration; + +public sealed class CiscoConnectorOptions : IValidatableObject +{ + public const string HttpClientName = "cisco-csaf"; + + /// + /// Endpoint for Cisco CSAF provider metadata discovery. + /// + [Required] + public string MetadataUri { get; set; } = "https://api.security.cisco.com/.well-known/csaf/provider-metadata.json"; + + /// + /// Optional bearer token used when Cisco endpoints require authentication. + /// + public string? ApiToken { get; set; } + + /// + /// How long provider metadata remains cached. + /// + public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromHours(6); + + /// + /// Whether to prefer offline snapshots when fetching metadata. + /// + public bool PreferOfflineSnapshot { get; set; } + + /// + /// When set, provider metadata will be persisted to the given file path. + /// + public bool PersistOfflineSnapshot { get; set; } + + public string? OfflineSnapshotPath { get; set; } + + public IEnumerable Validate(ValidationContext validationContext) + { + if (string.IsNullOrWhiteSpace(MetadataUri)) + { + yield return new ValidationResult("MetadataUri must be provided.", new[] { nameof(MetadataUri) }); + } + else if (!Uri.TryCreate(MetadataUri, UriKind.Absolute, out _)) + { + yield return new ValidationResult("MetadataUri must be an absolute URI.", new[] { nameof(MetadataUri) }); + } + + if (MetadataCacheDuration <= TimeSpan.Zero) + { + yield return new ValidationResult("MetadataCacheDuration must be greater than zero.", new[] { nameof(MetadataCacheDuration) }); + } + + if (PersistOfflineSnapshot && string.IsNullOrWhiteSpace(OfflineSnapshotPath)) + { + yield return new ValidationResult("OfflineSnapshotPath must be provided when PersistOfflineSnapshot is enabled.", new[] { nameof(OfflineSnapshotPath) }); + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptionsValidator.cs b/src/StellaOps.Vexer.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptionsValidator.cs new file mode 100644 index 00000000..6599b60a --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptionsValidator.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using StellaOps.Vexer.Connectors.Abstractions; + +namespace StellaOps.Vexer.Connectors.Cisco.CSAF.Configuration; + +public sealed class CiscoConnectorOptionsValidator : IVexConnectorOptionsValidator +{ + public void Validate(VexConnectorDescriptor descriptor, CiscoConnectorOptions options, IList errors) + { + ArgumentNullException.ThrowIfNull(descriptor); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(errors); + + var validationResults = new List(); + if (!Validator.TryValidateObject(options, new ValidationContext(options), validationResults, validateAllProperties: true)) + { + foreach (var result in validationResults) + { + errors.Add(result.ErrorMessage ?? "Cisco connector options validation failed."); + } + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/DependencyInjection/CiscoConnectorServiceCollectionExtensions.cs b/src/StellaOps.Vexer.Connectors.Cisco.CSAF/DependencyInjection/CiscoConnectorServiceCollectionExtensions.cs new file mode 100644 index 00000000..a8ce354c --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Cisco.CSAF/DependencyInjection/CiscoConnectorServiceCollectionExtensions.cs @@ -0,0 +1,52 @@ +using System.ComponentModel.DataAnnotations; +using System.Net.Http.Headers; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Vexer.Connectors.Cisco.CSAF.Configuration; +using StellaOps.Vexer.Connectors.Cisco.CSAF.Metadata; +using StellaOps.Vexer.Connectors.Abstractions; +using StellaOps.Vexer.Core; +using System.IO.Abstractions; + +namespace StellaOps.Vexer.Connectors.Cisco.CSAF.DependencyInjection; + +public static class CiscoConnectorServiceCollectionExtensions +{ + public static IServiceCollection AddCiscoCsafConnector(this IServiceCollection services, Action? configure = null) + { + ArgumentNullException.ThrowIfNull(services); + + services.AddOptions() + .Configure(options => + { + configure?.Invoke(options); + }) + .PostConfigure(options => + { + Validator.ValidateObject(options, new ValidationContext(options), validateAllProperties: true); + }); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddEnumerable(ServiceDescriptor.Singleton, CiscoConnectorOptionsValidator>()); + + services.AddHttpClient(CiscoConnectorOptions.HttpClientName) + .ConfigureHttpClient((provider, client) => + { + var options = provider.GetRequiredService>().Value; + client.Timeout = TimeSpan.FromSeconds(30); + client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); + if (!string.IsNullOrWhiteSpace(options.ApiToken)) + { + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", options.ApiToken); + } + }); + + services.AddSingleton(); + services.AddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/Metadata/CiscoProviderMetadataLoader.cs b/src/StellaOps.Vexer.Connectors.Cisco.CSAF/Metadata/CiscoProviderMetadataLoader.cs new file mode 100644 index 00000000..5d5d7438 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Cisco.CSAF/Metadata/CiscoProviderMetadataLoader.cs @@ -0,0 +1,332 @@ +using System.Collections.Immutable; +using System.Net; +using System.Net.Http.Headers; +using System.Text.Json; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Vexer.Connectors.Cisco.CSAF.Configuration; +using StellaOps.Vexer.Core; +using System.IO.Abstractions; + +namespace StellaOps.Vexer.Connectors.Cisco.CSAF.Metadata; + +public sealed class CiscoProviderMetadataLoader +{ + public const string CacheKey = "StellaOps.Vexer.Connectors.Cisco.CSAF.Metadata"; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly IMemoryCache _memoryCache; + private readonly ILogger _logger; + private readonly CiscoConnectorOptions _options; + private readonly IFileSystem _fileSystem; + private readonly JsonSerializerOptions _serializerOptions; + private readonly SemaphoreSlim _semaphore = new(1, 1); + + public CiscoProviderMetadataLoader( + IHttpClientFactory httpClientFactory, + IMemoryCache memoryCache, + IOptions options, + ILogger logger, + IFileSystem? fileSystem = null) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + ArgumentNullException.ThrowIfNull(options); + _options = options.Value ?? throw new ArgumentNullException(nameof(options)); + _fileSystem = fileSystem ?? new FileSystem(); + _serializerOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + }; + } + + public async Task LoadAsync(CancellationToken cancellationToken) + { + if (_memoryCache.TryGetValue(CacheKey, out var cached) && cached is not null && !cached.IsExpired()) + { + _logger.LogDebug("Returning cached Cisco provider metadata (expires {Expires}).", cached.ExpiresAt); + return new CiscoProviderMetadataResult(cached.Provider, cached.FetchedAt, cached.FromOffline, true); + } + + await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_memoryCache.TryGetValue(CacheKey, out cached) && cached is not null && !cached.IsExpired()) + { + return new CiscoProviderMetadataResult(cached.Provider, cached.FetchedAt, cached.FromOffline, true); + } + + CacheEntry? previous = cached; + + if (!_options.PreferOfflineSnapshot) + { + var network = await TryFetchFromNetworkAsync(previous, cancellationToken).ConfigureAwait(false); + if (network is not null) + { + StoreCache(network); + return new CiscoProviderMetadataResult(network.Provider, network.FetchedAt, false, false); + } + } + + var offline = TryLoadFromOffline(); + if (offline is not null) + { + var entry = offline with + { + FetchedAt = DateTimeOffset.UtcNow, + ExpiresAt = DateTimeOffset.UtcNow + _options.MetadataCacheDuration, + FromOffline = true, + }; + StoreCache(entry); + return new CiscoProviderMetadataResult(entry.Provider, entry.FetchedAt, true, false); + } + + throw new InvalidOperationException("Unable to load Cisco CSAF provider metadata from network or offline snapshot."); + } + finally + { + _semaphore.Release(); + } + } + + private async Task TryFetchFromNetworkAsync(CacheEntry? previous, CancellationToken cancellationToken) + { + try + { + var client = _httpClientFactory.CreateClient(CiscoConnectorOptions.HttpClientName); + using var request = new HttpRequestMessage(HttpMethod.Get, _options.MetadataUri); + if (!string.IsNullOrWhiteSpace(_options.ApiToken)) + { + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _options.ApiToken); + } + + if (!string.IsNullOrWhiteSpace(previous?.ETag) && EntityTagHeaderValue.TryParse(previous.ETag, out var etag)) + { + request.Headers.IfNoneMatch.Add(etag); + } + + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + if (response.StatusCode == HttpStatusCode.NotModified && previous is not null) + { + _logger.LogDebug("Cisco provider metadata not modified (etag {ETag}).", previous.ETag); + return previous with + { + FetchedAt = DateTimeOffset.UtcNow, + ExpiresAt = DateTimeOffset.UtcNow + _options.MetadataCacheDuration, + }; + } + + response.EnsureSuccessStatusCode(); + var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var provider = ParseProvider(payload); + var etagHeader = response.Headers.ETag?.ToString(); + + if (_options.PersistOfflineSnapshot && !string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath)) + { + try + { + _fileSystem.File.WriteAllText(_options.OfflineSnapshotPath, payload); + _logger.LogDebug("Persisted Cisco metadata snapshot to {Path}.", _options.OfflineSnapshotPath); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to persist Cisco metadata snapshot to {Path}.", _options.OfflineSnapshotPath); + } + } + + return new CacheEntry( + provider, + DateTimeOffset.UtcNow, + DateTimeOffset.UtcNow + _options.MetadataCacheDuration, + etagHeader, + FromOffline: false); + } + catch (Exception ex) when (ex is not OperationCanceledException && !_options.PreferOfflineSnapshot) + { + _logger.LogWarning(ex, "Failed to fetch Cisco provider metadata from {Uri}; falling back to offline snapshot when available.", _options.MetadataUri); + return null; + } + } + + private CacheEntry? TryLoadFromOffline() + { + if (string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath)) + { + return null; + } + + if (!_fileSystem.File.Exists(_options.OfflineSnapshotPath)) + { + _logger.LogWarning("Cisco offline snapshot path {Path} does not exist.", _options.OfflineSnapshotPath); + return null; + } + + try + { + var payload = _fileSystem.File.ReadAllText(_options.OfflineSnapshotPath); + var provider = ParseProvider(payload); + return new CacheEntry(provider, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow + _options.MetadataCacheDuration, null, true); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to load Cisco provider metadata from offline snapshot {Path}.", _options.OfflineSnapshotPath); + return null; + } + } + + private VexProvider ParseProvider(string payload) + { + if (string.IsNullOrWhiteSpace(payload)) + { + throw new InvalidOperationException("Cisco provider metadata payload was empty."); + } + + ProviderMetadataDocument? document; + try + { + document = JsonSerializer.Deserialize(payload, _serializerOptions); + } + catch (JsonException ex) + { + throw new InvalidOperationException("Failed to parse Cisco provider metadata.", ex); + } + + if (document?.Metadata?.Publisher?.ContactDetails is null || string.IsNullOrWhiteSpace(document.Metadata.Publisher.ContactDetails.Id)) + { + throw new InvalidOperationException("Cisco provider metadata did not include a publisher identifier."); + } + + var discovery = new VexProviderDiscovery(document.Discovery?.WellKnown, document.Discovery?.RolIe); + var trust = document.Trust is null + ? VexProviderTrust.Default + : new VexProviderTrust( + document.Trust.Weight ?? 1.0, + document.Trust.Cosign is null ? null : new VexCosignTrust(document.Trust.Cosign.Issuer ?? string.Empty, document.Trust.Cosign.IdentityPattern ?? string.Empty), + document.Trust.PgpFingerprints ?? Enumerable.Empty()); + + var directories = document.Distributions?.Directories is null + ? Enumerable.Empty() + : document.Distributions.Directories + .Where(static s => !string.IsNullOrWhiteSpace(s)) + .Select(static s => Uri.TryCreate(s, UriKind.Absolute, out var uri) ? uri : null) + .Where(static uri => uri is not null)! + .Select(static uri => uri!); + + return new VexProvider( + id: document.Metadata.Publisher.ContactDetails.Id, + displayName: document.Metadata.Publisher.Name ?? document.Metadata.Publisher.ContactDetails.Id, + kind: document.Metadata.Publisher.Category?.Equals("vendor", StringComparison.OrdinalIgnoreCase) == true ? VexProviderKind.Vendor : VexProviderKind.Hub, + baseUris: directories, + discovery: discovery, + trust: trust, + enabled: true); + } + + private void StoreCache(CacheEntry entry) + { + var options = new MemoryCacheEntryOptions + { + AbsoluteExpiration = entry.ExpiresAt, + }; + _memoryCache.Set(CacheKey, entry, options); + } + + private sealed record CacheEntry( + VexProvider Provider, + DateTimeOffset FetchedAt, + DateTimeOffset ExpiresAt, + string? ETag, + bool FromOffline) + { + public bool IsExpired() => DateTimeOffset.UtcNow >= ExpiresAt; + } +} + +public sealed record CiscoProviderMetadataResult( + VexProvider Provider, + DateTimeOffset FetchedAt, + bool FromOfflineSnapshot, + bool ServedFromCache); + +#region document models + +internal sealed class ProviderMetadataDocument +{ + [System.Text.Json.Serialization.JsonPropertyName("metadata")] + public ProviderMetadataMetadata Metadata { get; set; } = new(); + + [System.Text.Json.Serialization.JsonPropertyName("discovery")] + public ProviderMetadataDiscovery? Discovery { get; set; } + + [System.Text.Json.Serialization.JsonPropertyName("trust")] + public ProviderMetadataTrust? Trust { get; set; } + + [System.Text.Json.Serialization.JsonPropertyName("distributions")] + public ProviderMetadataDistributions? Distributions { get; set; } +} + +internal sealed class ProviderMetadataMetadata +{ + [System.Text.Json.Serialization.JsonPropertyName("publisher")] + public ProviderMetadataPublisher Publisher { get; set; } = new(); +} + +internal sealed class ProviderMetadataPublisher +{ + [System.Text.Json.Serialization.JsonPropertyName("name")] + public string? Name { get; set; } + + [System.Text.Json.Serialization.JsonPropertyName("category")] + public string? Category { get; set; } + + [System.Text.Json.Serialization.JsonPropertyName("contact_details")] + public ProviderMetadataPublisherContact ContactDetails { get; set; } = new(); +} + +internal sealed class ProviderMetadataPublisherContact +{ + [System.Text.Json.Serialization.JsonPropertyName("id")] + public string? Id { get; set; } +} + +internal sealed class ProviderMetadataDiscovery +{ + [System.Text.Json.Serialization.JsonPropertyName("well_known")] + public Uri? WellKnown { get; set; } + + [System.Text.Json.Serialization.JsonPropertyName("rolie")] + public Uri? RolIe { get; set; } +} + +internal sealed class ProviderMetadataTrust +{ + [System.Text.Json.Serialization.JsonPropertyName("weight")] + public double? Weight { get; set; } + + [System.Text.Json.Serialization.JsonPropertyName("cosign")] + public ProviderMetadataTrustCosign? Cosign { get; set; } + + [System.Text.Json.Serialization.JsonPropertyName("pgp_fingerprints")] + public string[]? PgpFingerprints { get; set; } +} + +internal sealed class ProviderMetadataTrustCosign +{ + [System.Text.Json.Serialization.JsonPropertyName("issuer")] + public string? Issuer { get; set; } + + [System.Text.Json.Serialization.JsonPropertyName("identity_pattern")] + public string? IdentityPattern { get; set; } +} + +internal sealed class ProviderMetadataDistributions +{ + [System.Text.Json.Serialization.JsonPropertyName("directories")] + public string[]? Directories { get; set; } +} + +#endregion diff --git a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/StellaOps.Vexer.Connectors.Cisco.CSAF.csproj b/src/StellaOps.Vexer.Connectors.Cisco.CSAF/StellaOps.Vexer.Connectors.Cisco.CSAF.csproj new file mode 100644 index 00000000..d7480a49 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Cisco.CSAF/StellaOps.Vexer.Connectors.Cisco.CSAF.csproj @@ -0,0 +1,20 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/TASKS.md b/src/StellaOps.Vexer.Connectors.Cisco.CSAF/TASKS.md index 1c751fc7..64bc0381 100644 --- a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/TASKS.md +++ b/src/StellaOps.Vexer.Connectors.Cisco.CSAF/TASKS.md @@ -2,6 +2,6 @@ If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and # TASKS | Task | Owner(s) | Depends on | Notes | |---|---|---|---| -|VEXER-CONN-CISCO-01-001 – Endpoint discovery & auth plumbing|Team Vexer Connectors – Cisco|VEXER-CONN-ABS-01-001|TODO – Resolve Cisco CSAF collection URLs, configure optional token auth, and validate discovery metadata for offline caching.| -|VEXER-CONN-CISCO-01-002 – CSAF pull loop & pagination|Team Vexer Connectors – Cisco|VEXER-CONN-CISCO-01-001, VEXER-STORAGE-01-003|TODO – Implement paginated fetch with retries/backoff, checksum validation, and raw document persistence.| +|VEXER-CONN-CISCO-01-001 – Endpoint discovery & auth plumbing|Team Vexer Connectors – Cisco|VEXER-CONN-ABS-01-001|**DONE (2025-10-17)** – Added `CiscoProviderMetadataLoader` with bearer token support, offline snapshot fallback, DI helpers, and tests covering network/offline discovery to unblock subsequent fetch work.| +|VEXER-CONN-CISCO-01-002 – CSAF pull loop & pagination|Team Vexer Connectors – Cisco|VEXER-CONN-CISCO-01-001, VEXER-STORAGE-01-003|**DONE (2025-10-17)** – Implemented paginated advisory fetch using provider directories, raw document persistence with dedupe/state tracking, offline resiliency, and unit coverage.| |VEXER-CONN-CISCO-01-003 – Provider trust metadata|Team Vexer Connectors – Cisco|VEXER-CONN-CISCO-01-002, VEXER-POLICY-01-001|TODO – Emit cosign/PGP trust metadata and advisory provenance hints for policy weighting.| diff --git a/src/StellaOps.Vexer.Connectors.MSRC.CSAF.Tests/Authentication/MsrcTokenProviderTests.cs b/src/StellaOps.Vexer.Connectors.MSRC.CSAF.Tests/Authentication/MsrcTokenProviderTests.cs new file mode 100644 index 00000000..cea0cb3c --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.MSRC.CSAF.Tests/Authentication/MsrcTokenProviderTests.cs @@ -0,0 +1,176 @@ +using System.Net; +using System.Net.Http; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NSubstitute; +using StellaOps.Vexer.Connectors.MSRC.CSAF.Authentication; +using StellaOps.Vexer.Connectors.MSRC.CSAF.Configuration; +using System.IO.Abstractions.TestingHelpers; +using Xunit; + +namespace StellaOps.Vexer.Connectors.MSRC.CSAF.Tests.Authentication; + +public sealed class MsrcTokenProviderTests +{ + [Fact] + public async Task GetAccessTokenAsync_CachesUntilExpiry() + { + var handler = new TestHttpMessageHandler(new[] + { + CreateTokenResponse("token-1"), + CreateTokenResponse("token-2"), + }); + var client = new HttpClient(handler) { BaseAddress = new Uri("https://login.microsoftonline.com/") }; + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var options = Options.Create(new MsrcConnectorOptions + { + TenantId = "contoso.onmicrosoft.com", + ClientId = "client-id", + ClientSecret = "secret", + }); + + var timeProvider = new AdjustableTimeProvider(); + var provider = new MsrcTokenProvider(factory, cache, fileSystem, options, NullLogger.Instance, timeProvider); + + var first = await provider.GetAccessTokenAsync(CancellationToken.None); + first.Value.Should().Be("token-1"); + handler.InvocationCount.Should().Be(1); + + var second = await provider.GetAccessTokenAsync(CancellationToken.None); + second.Value.Should().Be("token-1"); + handler.InvocationCount.Should().Be(1); + } + + [Fact] + public async Task GetAccessTokenAsync_RefreshesWhenExpired() + { + var handler = new TestHttpMessageHandler(new[] + { + CreateTokenResponse("token-1", expiresIn: 120), + CreateTokenResponse("token-2", expiresIn: 3600), + }); + var client = new HttpClient(handler) { BaseAddress = new Uri("https://login.microsoftonline.com/") }; + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var options = Options.Create(new MsrcConnectorOptions + { + TenantId = "contoso.onmicrosoft.com", + ClientId = "client-id", + ClientSecret = "secret", + ExpiryLeewaySeconds = 60, + }); + + var timeProvider = new AdjustableTimeProvider(); + var provider = new MsrcTokenProvider(factory, cache, fileSystem, options, NullLogger.Instance, timeProvider); + + var first = await provider.GetAccessTokenAsync(CancellationToken.None); + first.Value.Should().Be("token-1"); + handler.InvocationCount.Should().Be(1); + + timeProvider.Advance(TimeSpan.FromMinutes(2)); + var second = await provider.GetAccessTokenAsync(CancellationToken.None); + second.Value.Should().Be("token-2"); + handler.InvocationCount.Should().Be(2); + } + + [Fact] + public async Task GetAccessTokenAsync_OfflineStaticToken() + { + var factory = Substitute.For(); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var options = Options.Create(new MsrcConnectorOptions + { + PreferOfflineToken = true, + StaticAccessToken = "offline-token", + }); + + var provider = new MsrcTokenProvider(factory, cache, fileSystem, options, NullLogger.Instance); + var token = await provider.GetAccessTokenAsync(CancellationToken.None); + token.Value.Should().Be("offline-token"); + token.ExpiresAt.Should().Be(DateTimeOffset.MaxValue); + } + + [Fact] + public async Task GetAccessTokenAsync_OfflineFileToken() + { + var factory = Substitute.For(); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var offlinePath = fileSystem.Path.Combine("/tokens", "msrc.txt"); + fileSystem.AddFile(offlinePath, new MockFileData("file-token")); + + var options = Options.Create(new MsrcConnectorOptions + { + PreferOfflineToken = true, + OfflineTokenPath = offlinePath, + }); + + var provider = new MsrcTokenProvider(factory, cache, fileSystem, options, NullLogger.Instance); + var token = await provider.GetAccessTokenAsync(CancellationToken.None); + token.Value.Should().Be("file-token"); + token.ExpiresAt.Should().Be(DateTimeOffset.MaxValue); + } + + private static HttpResponseMessage CreateTokenResponse(string token, int expiresIn = 3600) + { + var json = $"{{\"access_token\":\"{token}\",\"token_type\":\"Bearer\",\"expires_in\":{expiresIn}}}"; + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }; + } + + private sealed class AdjustableTimeProvider : TimeProvider + { + private DateTimeOffset _now = DateTimeOffset.UtcNow; + + public override DateTimeOffset GetUtcNow() => _now; + + public void Advance(TimeSpan span) => _now = _now.Add(span); + } + + private sealed class SingleClientHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleClientHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class TestHttpMessageHandler : HttpMessageHandler + { + private readonly Queue _responses; + + public TestHttpMessageHandler(IEnumerable responses) + { + _responses = new Queue(responses); + } + + public int InvocationCount { get; private set; } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + InvocationCount++; + if (_responses.Count == 0) + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = new StringContent("no responses remaining"), + }); + } + + return Task.FromResult(_responses.Dequeue()); + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.MSRC.CSAF.Tests/StellaOps.Vexer.Connectors.MSRC.CSAF.Tests.csproj b/src/StellaOps.Vexer.Connectors.MSRC.CSAF.Tests/StellaOps.Vexer.Connectors.MSRC.CSAF.Tests.csproj new file mode 100644 index 00000000..b3eb395f --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.MSRC.CSAF.Tests/StellaOps.Vexer.Connectors.MSRC.CSAF.Tests.csproj @@ -0,0 +1,18 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Connectors.MSRC.CSAF/Authentication/MsrcTokenProvider.cs b/src/StellaOps.Vexer.Connectors.MSRC.CSAF/Authentication/MsrcTokenProvider.cs new file mode 100644 index 00000000..c201e8dd --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.MSRC.CSAF/Authentication/MsrcTokenProvider.cs @@ -0,0 +1,185 @@ +using System; +using System.Collections.Generic; +using System.IO.Abstractions; +using System.Net.Http; +using System.Net.Http.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Vexer.Connectors.MSRC.CSAF.Configuration; + +namespace StellaOps.Vexer.Connectors.MSRC.CSAF.Authentication; + +public interface IMsrcTokenProvider +{ + ValueTask GetAccessTokenAsync(CancellationToken cancellationToken); +} + +public sealed class MsrcTokenProvider : IMsrcTokenProvider, IDisposable +{ + private const string CachePrefix = "StellaOps.Vexer.Connectors.MSRC.CSAF.Token"; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly IMemoryCache _cache; + private readonly IFileSystem _fileSystem; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly MsrcConnectorOptions _options; + private readonly SemaphoreSlim _refreshLock = new(1, 1); + + public MsrcTokenProvider( + IHttpClientFactory httpClientFactory, + IMemoryCache cache, + IFileSystem fileSystem, + IOptions options, + ILogger logger, + TimeProvider? timeProvider = null) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _cache = cache ?? throw new ArgumentNullException(nameof(cache)); + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + ArgumentNullException.ThrowIfNull(options); + _options = options.Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(_fileSystem); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public async ValueTask GetAccessTokenAsync(CancellationToken cancellationToken) + { + if (_options.PreferOfflineToken) + { + return LoadOfflineToken(); + } + + var cacheKey = CreateCacheKey(); + if (_cache.TryGetValue(cacheKey, out var cachedToken) && + cachedToken is not null && + !cachedToken.IsExpired(_timeProvider.GetUtcNow())) + { + return cachedToken; + } + + await _refreshLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_cache.TryGetValue(cacheKey, out cachedToken) && + cachedToken is not null && + !cachedToken.IsExpired(_timeProvider.GetUtcNow())) + { + return cachedToken; + } + + var token = await RequestTokenAsync(cancellationToken).ConfigureAwait(false); + var absoluteExpiration = token.ExpiresAt == DateTimeOffset.MaxValue + ? (DateTimeOffset?)null + : token.ExpiresAt; + + var options = new MemoryCacheEntryOptions(); + if (absoluteExpiration.HasValue) + { + options.AbsoluteExpiration = absoluteExpiration.Value; + } + + _cache.Set(cacheKey, token, options); + return token; + } + finally + { + _refreshLock.Release(); + } + } + + private MsrcAccessToken LoadOfflineToken() + { + if (!string.IsNullOrWhiteSpace(_options.StaticAccessToken)) + { + return new MsrcAccessToken(_options.StaticAccessToken!, "Bearer", DateTimeOffset.MaxValue); + } + + if (string.IsNullOrWhiteSpace(_options.OfflineTokenPath)) + { + throw new InvalidOperationException("Offline token mode is enabled but no token was provided."); + } + + if (!_fileSystem.File.Exists(_options.OfflineTokenPath)) + { + throw new InvalidOperationException($"Offline token path '{_options.OfflineTokenPath}' does not exist."); + } + + var token = _fileSystem.File.ReadAllText(_options.OfflineTokenPath).Trim(); + if (string.IsNullOrEmpty(token)) + { + throw new InvalidOperationException("Offline token file was empty."); + } + + return new MsrcAccessToken(token, "Bearer", DateTimeOffset.MaxValue); + } + + private async Task RequestTokenAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("Fetching MSRC AAD access token for tenant {TenantId}.", _options.TenantId); + + var client = _httpClientFactory.CreateClient(MsrcConnectorOptions.TokenClientName); + using var request = new HttpRequestMessage(HttpMethod.Post, BuildTokenUri()) + { + Content = new FormUrlEncodedContent(new Dictionary + { + ["client_id"] = _options.ClientId, + ["client_secret"] = _options.ClientSecret!, + ["grant_type"] = "client_credentials", + ["scope"] = string.IsNullOrWhiteSpace(_options.Scope) ? MsrcConnectorOptions.DefaultScope : _options.Scope, + }), + }; + + using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException($"Failed to acquire MSRC access token ({(int)response.StatusCode}). Response: {payload}"); + } + + var tokenResponse = await response.Content.ReadFromJsonAsync(cancellationToken: cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException("Token endpoint returned an empty payload."); + + if (string.IsNullOrWhiteSpace(tokenResponse.AccessToken)) + { + throw new InvalidOperationException("Token endpoint response did not include an access_token."); + } + + var now = _timeProvider.GetUtcNow(); + var expiresAt = tokenResponse.ExpiresIn > _options.ExpiryLeewaySeconds + ? now.AddSeconds(tokenResponse.ExpiresIn - _options.ExpiryLeewaySeconds) + : now.AddMinutes(5); + + return new MsrcAccessToken(tokenResponse.AccessToken!, tokenResponse.TokenType ?? "Bearer", expiresAt); + } + + private string CreateCacheKey() + => $"{CachePrefix}:{_options.TenantId}:{_options.ClientId}:{_options.Scope}"; + + private Uri BuildTokenUri() + => new($"https://login.microsoftonline.com/{_options.TenantId}/oauth2/v2.0/token"); + + public void Dispose() => _refreshLock.Dispose(); + + private sealed record TokenResponse + { + [JsonPropertyName("access_token")] + public string? AccessToken { get; init; } + + [JsonPropertyName("token_type")] + public string? TokenType { get; init; } + + [JsonPropertyName("expires_in")] + public int ExpiresIn { get; init; } + } +} + +public sealed record MsrcAccessToken(string Value, string Type, DateTimeOffset ExpiresAt) +{ + public bool IsExpired(DateTimeOffset now) => now >= ExpiresAt; +} diff --git a/src/StellaOps.Vexer.Connectors.MSRC.CSAF/Configuration/MsrcConnectorOptions.cs b/src/StellaOps.Vexer.Connectors.MSRC.CSAF/Configuration/MsrcConnectorOptions.cs new file mode 100644 index 00000000..7139af96 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.MSRC.CSAF/Configuration/MsrcConnectorOptions.cs @@ -0,0 +1,95 @@ +using System; +using System.IO; +using System.IO.Abstractions; + +namespace StellaOps.Vexer.Connectors.MSRC.CSAF.Configuration; + +public sealed class MsrcConnectorOptions +{ + public const string TokenClientName = "vexer.connector.msrc.token"; + public const string DefaultScope = "https://api.msrc.microsoft.com/.default"; + + /// + /// Azure AD tenant identifier (GUID or domain). + /// + public string TenantId { get; set; } = string.Empty; + + /// + /// Azure AD application (client) identifier. + /// + public string ClientId { get; set; } = string.Empty; + + /// + /// Azure AD application secret for client credential flow. + /// + public string? ClientSecret { get; set; } + /// + /// OAuth scope requested for MSRC API access. + /// + public string Scope { get; set; } = DefaultScope; + + /// + /// When true, token acquisition is skipped and the connector expects offline handling. + /// + public bool PreferOfflineToken { get; set; } + /// + /// Optional path to a pre-provisioned bearer token used when is enabled. + /// + public string? OfflineTokenPath { get; set; } + /// + /// Optional fixed bearer token for constrained environments (e.g., short-lived offline bundles). + /// + public string? StaticAccessToken { get; set; } + /// + /// Minimum buffer (seconds) subtracted from token expiry before refresh. + /// + public int ExpiryLeewaySeconds { get; set; } = 60; + + public void Validate(IFileSystem? fileSystem = null) + { + if (PreferOfflineToken) + { + if (string.IsNullOrWhiteSpace(OfflineTokenPath) && string.IsNullOrWhiteSpace(StaticAccessToken)) + { + throw new InvalidOperationException("OfflineTokenPath or StaticAccessToken must be provided when PreferOfflineToken is enabled."); + } + } + else + { + if (string.IsNullOrWhiteSpace(TenantId)) + { + throw new InvalidOperationException("TenantId is required when not operating in offline token mode."); + } + + if (string.IsNullOrWhiteSpace(ClientId)) + { + throw new InvalidOperationException("ClientId is required when not operating in offline token mode."); + } + + if (string.IsNullOrWhiteSpace(ClientSecret)) + { + throw new InvalidOperationException("ClientSecret is required when not operating in offline token mode."); + } + } + + if (string.IsNullOrWhiteSpace(Scope)) + { + Scope = DefaultScope; + } + + if (ExpiryLeewaySeconds < 10) + { + ExpiryLeewaySeconds = 10; + } + + if (!string.IsNullOrWhiteSpace(OfflineTokenPath)) + { + var fs = fileSystem ?? new FileSystem(); + var directory = Path.GetDirectoryName(OfflineTokenPath); + if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory)) + { + fs.Directory.CreateDirectory(directory); + } + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.MSRC.CSAF/DependencyInjection/MsrcConnectorServiceCollectionExtensions.cs b/src/StellaOps.Vexer.Connectors.MSRC.CSAF/DependencyInjection/MsrcConnectorServiceCollectionExtensions.cs new file mode 100644 index 00000000..633e647c --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.MSRC.CSAF/DependencyInjection/MsrcConnectorServiceCollectionExtensions.cs @@ -0,0 +1,40 @@ +using System; +using System.Net; +using System.Net.Http; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using StellaOps.Vexer.Connectors.MSRC.CSAF.Authentication; +using StellaOps.Vexer.Connectors.MSRC.CSAF.Configuration; +using System.IO.Abstractions; + +namespace StellaOps.Vexer.Connectors.MSRC.CSAF.DependencyInjection; + +public static class MsrcConnectorServiceCollectionExtensions +{ + public static IServiceCollection AddMsrcCsafConnector(this IServiceCollection services, Action? configure = null) + { + ArgumentNullException.ThrowIfNull(services); + + services.TryAddSingleton(); + services.TryAddSingleton(); + + services.AddOptions() + .Configure(options => configure?.Invoke(options)); + + services.AddHttpClient(MsrcConnectorOptions.TokenClientName, client => + { + client.Timeout = TimeSpan.FromSeconds(30); + client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Vexer.Connectors.MSRC.CSAF/1.0"); + client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); + }) + .ConfigurePrimaryHttpMessageHandler(static () => new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.All, + }); + + services.AddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Vexer.Connectors.MSRC.CSAF/StellaOps.Vexer.Connectors.MSRC.CSAF.csproj b/src/StellaOps.Vexer.Connectors.MSRC.CSAF/StellaOps.Vexer.Connectors.MSRC.CSAF.csproj new file mode 100644 index 00000000..a99a942f --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.MSRC.CSAF/StellaOps.Vexer.Connectors.MSRC.CSAF.csproj @@ -0,0 +1,18 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Connectors.MSRC.CSAF/TASKS.md b/src/StellaOps.Vexer.Connectors.MSRC.CSAF/TASKS.md index fbd8d2aa..97b7328a 100644 --- a/src/StellaOps.Vexer.Connectors.MSRC.CSAF/TASKS.md +++ b/src/StellaOps.Vexer.Connectors.MSRC.CSAF/TASKS.md @@ -2,6 +2,6 @@ If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and # TASKS | Task | Owner(s) | Depends on | Notes | |---|---|---|---| -|VEXER-CONN-MS-01-001 – AAD onboarding & token cache|Team Vexer Connectors – MSRC|VEXER-CONN-ABS-01-001|TODO – Implement Azure AD credential flow, token caching, and validation for MSRC CSAF access with offline fallback guidance.| +|VEXER-CONN-MS-01-001 – AAD onboarding & token cache|Team Vexer Connectors – MSRC|VEXER-CONN-ABS-01-001|**DONE (2025-10-17)** – Added MSRC connector project with configurable AAD options, token provider (offline/online modes), DI wiring, and unit tests covering caching and fallback scenarios.| |VEXER-CONN-MS-01-002 – CSAF download pipeline|Team Vexer Connectors – MSRC|VEXER-CONN-MS-01-001, VEXER-STORAGE-01-003|TODO – Fetch CSAF packages with retry/backoff, checksum verification, and raw document persistence plus quarantine for schema failures.| |VEXER-CONN-MS-01-003 – Trust metadata & provenance hints|Team Vexer Connectors – MSRC|VEXER-CONN-MS-01-002, VEXER-POLICY-01-001|TODO – Emit cosign/AAD issuer metadata, attach provenance details, and document policy integration.| diff --git a/src/StellaOps.Vexer.Connectors.Oracle.CSAF.Tests/Metadata/OracleCatalogLoaderTests.cs b/src/StellaOps.Vexer.Connectors.Oracle.CSAF.Tests/Metadata/OracleCatalogLoaderTests.cs new file mode 100644 index 00000000..de05d2d4 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Oracle.CSAF.Tests/Metadata/OracleCatalogLoaderTests.cs @@ -0,0 +1,205 @@ +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Vexer.Connectors.Oracle.CSAF.Configuration; +using StellaOps.Vexer.Connectors.Oracle.CSAF.Metadata; +using System.IO.Abstractions.TestingHelpers; +using Xunit; +using System.Threading; + +namespace StellaOps.Vexer.Connectors.Oracle.CSAF.Tests.Metadata; + +public sealed class OracleCatalogLoaderTests +{ + private const string SampleCatalog = """ + { + "generated": "2025-09-30T18:00:00Z", + "catalog": [ + { + "id": "CPU2025Oct", + "title": "Oracle Critical Patch Update Advisory - October 2025", + "published": "2025-10-15T00:00:00Z", + "revision": "2025-10-15T00:00:00Z", + "document": { + "url": "https://updates.oracle.com/cpu/2025-10/cpu2025oct.json", + "sha256": "abc123", + "size": 1024 + }, + "products": ["Oracle Database", "Java SE"] + } + ], + "schedule": [ + { "window": "2025-Oct", "releaseDate": "2025-10-15T00:00:00Z" } + ] + } + """; + + private const string SampleCalendar = """ + { + "cpuWindows": [ + { "name": "2026-Jan", "releaseDate": "2026-01-21T00:00:00Z" } + ] + } + """; + + [Fact] + public async Task LoadAsync_FetchesAndCachesCatalog() + { + var handler = new TestHttpMessageHandler(new Dictionary + { + [new Uri("https://www.oracle.com/security-alerts/cpu/csaf/catalog.json")] = CreateResponse(SampleCatalog), + [new Uri("https://www.oracle.com/security-alerts/cpu/cal.json")] = CreateResponse(SampleCalendar), + }); + var client = new HttpClient(handler); + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var loader = new OracleCatalogLoader(factory, cache, fileSystem, NullLogger.Instance, new AdjustableTimeProvider()); + + var options = new OracleConnectorOptions + { + CatalogUri = new Uri("https://www.oracle.com/security-alerts/cpu/csaf/catalog.json"), + CpuCalendarUri = new Uri("https://www.oracle.com/security-alerts/cpu/cal.json"), + OfflineSnapshotPath = "/snapshots/oracle-catalog.json", + }; + + var result = await loader.LoadAsync(options, CancellationToken.None); + result.Metadata.Entries.Should().HaveCount(1); + result.Metadata.CpuSchedule.Should().Contain(r => r.Window == "2025-Oct"); + result.Metadata.CpuSchedule.Should().Contain(r => r.Window == "2026-Jan"); + result.FromCache.Should().BeFalse(); + fileSystem.FileExists(options.OfflineSnapshotPath!).Should().BeTrue(); + + handler.ResetInvocationCount(); + var cached = await loader.LoadAsync(options, CancellationToken.None); + cached.FromCache.Should().BeTrue(); + handler.InvocationCount.Should().Be(0); + } + + [Fact] + public async Task LoadAsync_UsesOfflineSnapshotWhenNetworkFails() + { + var handler = new TestHttpMessageHandler(new Dictionary()); + var client = new HttpClient(handler); + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var offlineJson = """ + { + "metadata": { + "generatedAt": "2025-09-30T18:00:00Z", + "entries": [ + { + "id": "CPU2025Oct", + "title": "Oracle Critical Patch Update Advisory - October 2025", + "documentUri": "https://updates.oracle.com/cpu/2025-10/cpu2025oct.json", + "publishedAt": "2025-10-15T00:00:00Z", + "revision": "2025-10-15T00:00:00Z", + "sha256": "abc123", + "size": 1024, + "products": [ "Oracle Database" ] + } + ], + "cpuSchedule": [ + { "window": "2025-Oct", "releaseDate": "2025-10-15T00:00:00Z" } + ] + }, + "fetchedAt": "2025-10-01T00:00:00Z" + } + """; + fileSystem.AddFile("/snapshots/oracle-catalog.json", new MockFileData(offlineJson)); + var loader = new OracleCatalogLoader(factory, cache, fileSystem, NullLogger.Instance, new AdjustableTimeProvider()); + + var options = new OracleConnectorOptions + { + CatalogUri = new Uri("https://www.oracle.com/security-alerts/cpu/csaf/catalog.json"), + OfflineSnapshotPath = "/snapshots/oracle-catalog.json", + PreferOfflineSnapshot = true, + }; + + var result = await loader.LoadAsync(options, CancellationToken.None); + result.FromOfflineSnapshot.Should().BeTrue(); + result.Metadata.Entries.Should().NotBeEmpty(); + } + + [Fact] + public async Task LoadAsync_ThrowsWhenOfflinePreferredButMissing() + { + var handler = new TestHttpMessageHandler(new Dictionary()); + var client = new HttpClient(handler); + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var loader = new OracleCatalogLoader(factory, cache, fileSystem, NullLogger.Instance); + + var options = new OracleConnectorOptions + { + CatalogUri = new Uri("https://www.oracle.com/security-alerts/cpu/csaf/catalog.json"), + PreferOfflineSnapshot = true, + OfflineSnapshotPath = "/missing/oracle-catalog.json", + }; + + await Assert.ThrowsAsync(() => loader.LoadAsync(options, CancellationToken.None)); + } + + private static HttpResponseMessage CreateResponse(string payload) + => new(HttpStatusCode.OK) + { + Content = new StringContent(payload, Encoding.UTF8, "application/json"), + }; + + private sealed class SingleClientHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleClientHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class AdjustableTimeProvider : TimeProvider + { + private DateTimeOffset _now = DateTimeOffset.UtcNow; + + public override DateTimeOffset GetUtcNow() => _now; + } + + private sealed class TestHttpMessageHandler : HttpMessageHandler + { + private readonly Dictionary _responses; + + public TestHttpMessageHandler(Dictionary responses) + { + _responses = responses; + } + + public int InvocationCount { get; private set; } + + public void ResetInvocationCount() => InvocationCount = 0; + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + InvocationCount++; + if (request.RequestUri is not null && _responses.TryGetValue(request.RequestUri, out var response)) + { + var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + return new HttpResponseMessage(response.StatusCode) + { + Content = new StringContent(payload, Encoding.UTF8, "application/json"), + }; + } + + return new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = new StringContent("unexpected request"), + }; + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.Oracle.CSAF.Tests/StellaOps.Vexer.Connectors.Oracle.CSAF.Tests.csproj b/src/StellaOps.Vexer.Connectors.Oracle.CSAF.Tests/StellaOps.Vexer.Connectors.Oracle.CSAF.Tests.csproj new file mode 100644 index 00000000..78a8cb0a --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Oracle.CSAF.Tests/StellaOps.Vexer.Connectors.Oracle.CSAF.Tests.csproj @@ -0,0 +1,17 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptions.cs b/src/StellaOps.Vexer.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptions.cs new file mode 100644 index 00000000..aea25b80 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptions.cs @@ -0,0 +1,85 @@ +using System; +using System.IO; +using System.IO.Abstractions; + +namespace StellaOps.Vexer.Connectors.Oracle.CSAF.Configuration; + +public sealed class OracleConnectorOptions +{ + public const string HttpClientName = "vexer.connector.oracle.catalog"; + + /// + /// Oracle CSAF catalog endpoint hosting advisory metadata. + /// + public Uri CatalogUri { get; set; } = new("https://www.oracle.com/security-alerts/cpu/csaf/catalog.json"); + + /// + /// Optional CPU calendar endpoint providing upcoming release dates. + /// + public Uri? CpuCalendarUri { get; set; } + /// + /// Duration the discovery metadata should be cached before refresh. + /// + public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromHours(6); + + /// + /// When true, the loader will prefer offline snapshot data over network fetches. + /// + public bool PreferOfflineSnapshot { get; set; } + /// + /// Optional file path for persisting or ingesting catalog snapshots. + /// + public string? OfflineSnapshotPath { get; set; } + /// + /// Enables writing fresh catalog responses to . + /// + public bool PersistOfflineSnapshot { get; set; } = true; + + /// + /// Optional request delay when iterating catalogue entries (for rate limiting). + /// + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); + + public void Validate(IFileSystem? fileSystem = null) + { + if (CatalogUri is null || !CatalogUri.IsAbsoluteUri) + { + throw new InvalidOperationException("CatalogUri must be an absolute URI."); + } + + if (CatalogUri.Scheme is not ("http" or "https")) + { + throw new InvalidOperationException("CatalogUri must use HTTP or HTTPS."); + } + + if (CpuCalendarUri is not null && (!CpuCalendarUri.IsAbsoluteUri || CpuCalendarUri.Scheme is not ("http" or "https"))) + { + throw new InvalidOperationException("CpuCalendarUri must be an absolute HTTP(S) URI when provided."); + } + + if (MetadataCacheDuration <= TimeSpan.Zero) + { + throw new InvalidOperationException("MetadataCacheDuration must be positive."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("RequestDelay cannot be negative."); + } + + if (PreferOfflineSnapshot && string.IsNullOrWhiteSpace(OfflineSnapshotPath)) + { + throw new InvalidOperationException("OfflineSnapshotPath must be provided when PreferOfflineSnapshot is enabled."); + } + + if (!string.IsNullOrWhiteSpace(OfflineSnapshotPath)) + { + var fs = fileSystem ?? new FileSystem(); + var directory = Path.GetDirectoryName(OfflineSnapshotPath); + if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory)) + { + fs.Directory.CreateDirectory(directory); + } + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptionsValidator.cs b/src/StellaOps.Vexer.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptionsValidator.cs new file mode 100644 index 00000000..13574413 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptionsValidator.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using System.IO.Abstractions; +using StellaOps.Vexer.Connectors.Abstractions; + +namespace StellaOps.Vexer.Connectors.Oracle.CSAF.Configuration; + +public sealed class OracleConnectorOptionsValidator : IVexConnectorOptionsValidator +{ + private readonly IFileSystem _fileSystem; + + public OracleConnectorOptionsValidator(IFileSystem fileSystem) + { + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + } + + public void Validate(VexConnectorDescriptor descriptor, OracleConnectorOptions options, IList errors) + { + ArgumentNullException.ThrowIfNull(descriptor); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(errors); + + try + { + options.Validate(_fileSystem); + } + catch (Exception ex) + { + errors.Add(ex.Message); + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/DependencyInjection/OracleConnectorServiceCollectionExtensions.cs b/src/StellaOps.Vexer.Connectors.Oracle.CSAF/DependencyInjection/OracleConnectorServiceCollectionExtensions.cs new file mode 100644 index 00000000..af8aa641 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Oracle.CSAF/DependencyInjection/OracleConnectorServiceCollectionExtensions.cs @@ -0,0 +1,45 @@ +using System; +using System.Net; +using System.Net.Http; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using StellaOps.Vexer.Connectors.Abstractions; +using StellaOps.Vexer.Connectors.Oracle.CSAF.Configuration; +using StellaOps.Vexer.Connectors.Oracle.CSAF.Metadata; +using StellaOps.Vexer.Core; +using System.IO.Abstractions; + +namespace StellaOps.Vexer.Connectors.Oracle.CSAF.DependencyInjection; + +public static class OracleConnectorServiceCollectionExtensions +{ + public static IServiceCollection AddOracleCsafConnector(this IServiceCollection services, Action? configure = null) + { + ArgumentNullException.ThrowIfNull(services); + + services.TryAddSingleton(); + services.TryAddSingleton(); + + services.AddOptions() + .Configure(options => configure?.Invoke(options)); + + services.AddSingleton, OracleConnectorOptionsValidator>(); + + services.AddHttpClient(OracleConnectorOptions.HttpClientName, client => + { + client.Timeout = TimeSpan.FromSeconds(60); + client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Vexer.Connectors.Oracle.CSAF/1.0"); + client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); + }) + .ConfigurePrimaryHttpMessageHandler(static () => new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.All, + }); + + services.AddSingleton(); + services.AddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/Metadata/OracleCatalogLoader.cs b/src/StellaOps.Vexer.Connectors.Oracle.CSAF/Metadata/OracleCatalogLoader.cs new file mode 100644 index 00000000..8268aff2 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Oracle.CSAF/Metadata/OracleCatalogLoader.cs @@ -0,0 +1,418 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO.Abstractions; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using StellaOps.Vexer.Connectors.Oracle.CSAF.Configuration; + +namespace StellaOps.Vexer.Connectors.Oracle.CSAF.Metadata; + +public sealed class OracleCatalogLoader +{ + public const string CachePrefix = "StellaOps.Vexer.Connectors.Oracle.CSAF.Catalog"; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly IMemoryCache _memoryCache; + private readonly IFileSystem _fileSystem; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly SemaphoreSlim _semaphore = new(1, 1); + private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web); + + public OracleCatalogLoader( + IHttpClientFactory httpClientFactory, + IMemoryCache memoryCache, + IFileSystem fileSystem, + ILogger logger, + TimeProvider? timeProvider = null) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache)); + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public async Task LoadAsync(OracleConnectorOptions options, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + options.Validate(_fileSystem); + + var cacheKey = CreateCacheKey(options); + if (_memoryCache.TryGetValue(cacheKey, out var cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow())) + { + return cached.ToResult(fromCache: true); + } + + await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_memoryCache.TryGetValue(cacheKey, out cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow())) + { + return cached.ToResult(fromCache: true); + } + + CacheEntry? entry = null; + if (options.PreferOfflineSnapshot) + { + entry = LoadFromOffline(options); + if (entry is null) + { + throw new InvalidOperationException("PreferOfflineSnapshot is enabled but no offline snapshot was found or could be loaded."); + } + } + else + { + entry = await TryFetchFromNetworkAsync(options, cancellationToken).ConfigureAwait(false) + ?? LoadFromOffline(options); + } + + if (entry is null) + { + throw new InvalidOperationException("Unable to load Oracle CSAF catalog from network or offline snapshot."); + } + + var expiration = entry.MetadataCacheDuration == TimeSpan.Zero + ? (DateTimeOffset?)null + : _timeProvider.GetUtcNow().Add(entry.MetadataCacheDuration); + + var cacheEntryOptions = new MemoryCacheEntryOptions(); + if (expiration.HasValue) + { + cacheEntryOptions.AbsoluteExpiration = expiration.Value; + } + + _memoryCache.Set(cacheKey, entry with { CachedAt = _timeProvider.GetUtcNow() }, cacheEntryOptions); + return entry.ToResult(fromCache: false); + } + finally + { + _semaphore.Release(); + } + } + + private async Task TryFetchFromNetworkAsync(OracleConnectorOptions options, CancellationToken cancellationToken) + { + try + { + var client = _httpClientFactory.CreateClient(OracleConnectorOptions.HttpClientName); + using var response = await client.GetAsync(options.CatalogUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + var catalogPayload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + + string? calendarPayload = null; + if (options.CpuCalendarUri is not null) + { + try + { + using var calendarResponse = await client.GetAsync(options.CpuCalendarUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + calendarResponse.EnsureSuccessStatusCode(); + calendarPayload = await calendarResponse.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, "Failed to fetch Oracle CPU calendar from {Uri}; continuing without schedule.", options.CpuCalendarUri); + } + } + + var metadata = ParseMetadata(catalogPayload, calendarPayload); + var fetchedAt = _timeProvider.GetUtcNow(); + var entry = new CacheEntry(metadata, fetchedAt, fetchedAt, options.MetadataCacheDuration, false); + + PersistSnapshotIfNeeded(options, metadata, fetchedAt); + return entry; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, "Failed to fetch Oracle CSAF catalog from {Uri}; attempting offline fallback if available.", options.CatalogUri); + return null; + } + } + + private CacheEntry? LoadFromOffline(OracleConnectorOptions options) + { + if (string.IsNullOrWhiteSpace(options.OfflineSnapshotPath)) + { + return null; + } + + if (!_fileSystem.File.Exists(options.OfflineSnapshotPath)) + { + _logger.LogWarning("Oracle offline snapshot path {Path} does not exist.", options.OfflineSnapshotPath); + return null; + } + + try + { + var payload = _fileSystem.File.ReadAllText(options.OfflineSnapshotPath); + var snapshot = JsonSerializer.Deserialize(payload, _serializerOptions); + if (snapshot is null) + { + throw new InvalidOperationException("Offline snapshot payload was empty."); + } + + return new CacheEntry(snapshot.Metadata, snapshot.FetchedAt, _timeProvider.GetUtcNow(), options.MetadataCacheDuration, true); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to load Oracle CSAF catalog from offline snapshot {Path}.", options.OfflineSnapshotPath); + return null; + } + } + + private OracleCatalogMetadata ParseMetadata(string catalogPayload, string? calendarPayload) + { + if (string.IsNullOrWhiteSpace(catalogPayload)) + { + throw new InvalidOperationException("Oracle catalog payload was empty."); + } + + using var document = JsonDocument.Parse(catalogPayload); + var root = document.RootElement; + + var generatedAt = root.TryGetProperty("generated", out var generatedElement) && generatedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(generatedElement.GetString(), out var generated) + ? generated + : _timeProvider.GetUtcNow(); + + var entries = ParseEntries(root); + var schedule = ParseSchedule(root); + + if (!string.IsNullOrWhiteSpace(calendarPayload)) + { + schedule = MergeSchedule(schedule, calendarPayload); + } + + return new OracleCatalogMetadata(generatedAt, entries, schedule); + } + + private ImmutableArray ParseEntries(JsonElement root) + { + if (!root.TryGetProperty("catalog", out var catalogElement) || catalogElement.ValueKind is not JsonValueKind.Array) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var entry in catalogElement.EnumerateArray()) + { + var id = entry.TryGetProperty("id", out var idElement) && idElement.ValueKind == JsonValueKind.String ? idElement.GetString() : null; + var title = entry.TryGetProperty("title", out var titleElement) && titleElement.ValueKind == JsonValueKind.String ? titleElement.GetString() : null; + if (string.IsNullOrWhiteSpace(id) || string.IsNullOrWhiteSpace(title)) + { + continue; + } + + DateTimeOffset publishedAt = default; + if (entry.TryGetProperty("published", out var publishedElement) && publishedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(publishedElement.GetString(), out var publishedParsed)) + { + publishedAt = publishedParsed; + } + + string? revision = null; + if (entry.TryGetProperty("revision", out var revisionElement) && revisionElement.ValueKind == JsonValueKind.String) + { + revision = revisionElement.GetString(); + } + + ImmutableArray products = ImmutableArray.Empty; + if (entry.TryGetProperty("products", out var productsElement)) + { + products = ParseStringArray(productsElement); + } + + Uri? documentUri = null; + string? sha256 = null; + long? size = null; + + if (entry.TryGetProperty("document", out var documentElement) && documentElement.ValueKind == JsonValueKind.Object) + { + if (documentElement.TryGetProperty("url", out var urlElement) && urlElement.ValueKind == JsonValueKind.String && Uri.TryCreate(urlElement.GetString(), UriKind.Absolute, out var parsedUri)) + { + documentUri = parsedUri; + } + + if (documentElement.TryGetProperty("sha256", out var hashElement) && hashElement.ValueKind == JsonValueKind.String) + { + sha256 = hashElement.GetString(); + } + + if (documentElement.TryGetProperty("size", out var sizeElement) && sizeElement.ValueKind == JsonValueKind.Number && sizeElement.TryGetInt64(out var parsedSize)) + { + size = parsedSize; + } + } + + if (documentUri is null) + { + continue; + } + + builder.Add(new OracleCatalogEntry(id!, title!, documentUri, publishedAt, revision, sha256, size, products)); + } + + return builder.ToImmutable(); + } + + private ImmutableArray ParseSchedule(JsonElement root) + { + if (!root.TryGetProperty("schedule", out var scheduleElement) || scheduleElement.ValueKind is not JsonValueKind.Array) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var item in scheduleElement.EnumerateArray()) + { + var window = item.TryGetProperty("window", out var windowElement) && windowElement.ValueKind == JsonValueKind.String ? windowElement.GetString() : null; + if (string.IsNullOrWhiteSpace(window)) + { + continue; + } + + DateTimeOffset releaseDate = default; + if (item.TryGetProperty("releaseDate", out var releaseElement) && releaseElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(releaseElement.GetString(), out var parsed)) + { + releaseDate = parsed; + } + + builder.Add(new OracleCpuRelease(window!, releaseDate)); + } + + return builder.ToImmutable(); + } + + private ImmutableArray MergeSchedule(ImmutableArray existing, string calendarPayload) + { + try + { + using var document = JsonDocument.Parse(calendarPayload); + var root = document.RootElement; + if (!root.TryGetProperty("cpuWindows", out var windowsElement) || windowsElement.ValueKind is not JsonValueKind.Array) + { + return existing; + } + + var builder = existing.ToBuilder(); + var known = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var item in builder) + { + known.Add(item.Window); + } + + foreach (var windowElement in windowsElement.EnumerateArray()) + { + var name = windowElement.TryGetProperty("name", out var nameElement) && nameElement.ValueKind == JsonValueKind.String ? nameElement.GetString() : null; + if (string.IsNullOrWhiteSpace(name)) + { + continue; + } + + if (!known.Add(name)) + { + continue; + } + + DateTimeOffset releaseDate = default; + if (windowElement.TryGetProperty("releaseDate", out var releaseElement) && releaseElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(releaseElement.GetString(), out var parsed)) + { + releaseDate = parsed; + } + + builder.Add(new OracleCpuRelease(name!, releaseDate)); + } + + return builder.ToImmutable(); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, "Failed to parse Oracle CPU calendar payload; continuing with existing schedule data."); + return existing; + } + } + + private ImmutableArray ParseStringArray(JsonElement element) + { + if (element.ValueKind is not JsonValueKind.Array) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var item in element.EnumerateArray()) + { + if (item.ValueKind == JsonValueKind.String) + { + var value = item.GetString(); + if (!string.IsNullOrWhiteSpace(value)) + { + builder.Add(value); + } + } + } + + return builder.ToImmutable(); + } + + private void PersistSnapshotIfNeeded(OracleConnectorOptions options, OracleCatalogMetadata metadata, DateTimeOffset fetchedAt) + { + if (!options.PersistOfflineSnapshot || string.IsNullOrWhiteSpace(options.OfflineSnapshotPath)) + { + return; + } + + try + { + var snapshot = new OracleCatalogSnapshot(metadata, fetchedAt); + var payload = JsonSerializer.Serialize(snapshot, _serializerOptions); + _fileSystem.File.WriteAllText(options.OfflineSnapshotPath, payload); + _logger.LogDebug("Persisted Oracle CSAF catalog snapshot to {Path}.", options.OfflineSnapshotPath); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to persist Oracle CSAF catalog snapshot to {Path}.", options.OfflineSnapshotPath); + } + } + + private static string CreateCacheKey(OracleConnectorOptions options) + => $"{CachePrefix}:{options.CatalogUri}:{options.CpuCalendarUri}"; + + private sealed record CacheEntry(OracleCatalogMetadata Metadata, DateTimeOffset FetchedAt, DateTimeOffset CachedAt, TimeSpan MetadataCacheDuration, bool FromOfflineSnapshot) + { + public bool IsExpired(DateTimeOffset now) + => MetadataCacheDuration > TimeSpan.Zero && now >= CachedAt + MetadataCacheDuration; + + public OracleCatalogResult ToResult(bool fromCache) + => new(Metadata, FetchedAt, fromCache, FromOfflineSnapshot); + } + + private sealed record OracleCatalogSnapshot(OracleCatalogMetadata Metadata, DateTimeOffset FetchedAt); +} + +public sealed record OracleCatalogMetadata( + DateTimeOffset GeneratedAt, + ImmutableArray Entries, + ImmutableArray CpuSchedule); + +public sealed record OracleCatalogEntry( + string Id, + string Title, + Uri DocumentUri, + DateTimeOffset PublishedAt, + string? Revision, + string? Sha256, + long? Size, + ImmutableArray Products); + +public sealed record OracleCpuRelease(string Window, DateTimeOffset ReleaseDate); + +public sealed record OracleCatalogResult( + OracleCatalogMetadata Metadata, + DateTimeOffset FetchedAt, + bool FromCache, + bool FromOfflineSnapshot); diff --git a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/OracleCsafConnector.cs b/src/StellaOps.Vexer.Connectors.Oracle.CSAF/OracleCsafConnector.cs new file mode 100644 index 00000000..ec4bb0e3 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Oracle.CSAF/OracleCsafConnector.cs @@ -0,0 +1,81 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.Logging; +using StellaOps.Vexer.Connectors.Abstractions; +using StellaOps.Vexer.Connectors.Oracle.CSAF.Configuration; +using StellaOps.Vexer.Connectors.Oracle.CSAF.Metadata; +using StellaOps.Vexer.Core; + +namespace StellaOps.Vexer.Connectors.Oracle.CSAF; + +public sealed class OracleCsafConnector : VexConnectorBase +{ + private static readonly VexConnectorDescriptor DescriptorInstance = new( + id: "vexer:oracle", + kind: VexProviderKind.Vendor, + displayName: "Oracle CSAF") + { + Tags = ImmutableArray.Create("oracle", "csaf", "cpu"), + }; + + private readonly OracleCatalogLoader _catalogLoader; + private readonly IEnumerable> _validators; + + private OracleConnectorOptions? _options; + private OracleCatalogResult? _catalog; + + public OracleCsafConnector( + OracleCatalogLoader catalogLoader, + IEnumerable> validators, + ILogger logger, + TimeProvider timeProvider) + : base(DescriptorInstance, logger, timeProvider) + { + _catalogLoader = catalogLoader ?? throw new ArgumentNullException(nameof(catalogLoader)); + _validators = validators ?? Array.Empty>(); + } + + public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) + { + _options = VexConnectorOptionsBinder.Bind( + Descriptor, + settings, + validators: _validators); + + _catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); + LogConnectorEvent(LogLevel.Information, "validate", "Oracle CSAF catalogue loaded.", new Dictionary + { + ["catalogEntryCount"] = _catalog.Metadata.Entries.Length, + ["scheduleCount"] = _catalog.Metadata.CpuSchedule.Length, + ["fromOffline"] = _catalog.FromOfflineSnapshot, + }); + } + + public override async IAsyncEnumerable FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + if (_options is null) + { + throw new InvalidOperationException("Connector must be validated before fetch operations."); + } + + if (_catalog is null) + { + _catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); + } + + LogConnectorEvent(LogLevel.Debug, "fetch", "Oracle CSAF discovery ready; document ingestion handled by follow-up task.", new Dictionary + { + ["since"] = context.Since?.ToString("O"), + }); + + yield break; + } + + public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => throw new NotSupportedException("OracleCsafConnector relies on dedicated CSAF normalizers."); + + public OracleCatalogResult? GetCachedCatalog() => _catalog; +} diff --git a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/StellaOps.Vexer.Connectors.Oracle.CSAF.csproj b/src/StellaOps.Vexer.Connectors.Oracle.CSAF/StellaOps.Vexer.Connectors.Oracle.CSAF.csproj new file mode 100644 index 00000000..a99a942f --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Oracle.CSAF/StellaOps.Vexer.Connectors.Oracle.CSAF.csproj @@ -0,0 +1,18 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/TASKS.md b/src/StellaOps.Vexer.Connectors.Oracle.CSAF/TASKS.md index fc768ce8..69452a71 100644 --- a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/TASKS.md +++ b/src/StellaOps.Vexer.Connectors.Oracle.CSAF/TASKS.md @@ -2,6 +2,6 @@ If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and # TASKS | Task | Owner(s) | Depends on | Notes | |---|---|---|---| -|VEXER-CONN-ORACLE-01-001 – Oracle CSAF catalogue discovery|Team Vexer Connectors – Oracle|VEXER-CONN-ABS-01-001|TODO – Implement catalogue discovery, CPU calendar awareness, and offline snapshot import for Oracle CSAF feeds.| +|VEXER-CONN-ORACLE-01-001 – Oracle CSAF catalogue discovery|Team Vexer Connectors – Oracle|VEXER-CONN-ABS-01-001|DOING (2025-10-17) – Implement catalogue discovery, CPU calendar awareness, and offline snapshot import for Oracle CSAF feeds.| |VEXER-CONN-ORACLE-01-002 – CSAF download & dedupe pipeline|Team Vexer Connectors – Oracle|VEXER-CONN-ORACLE-01-001, VEXER-STORAGE-01-003|TODO – Fetch CSAF documents with retry/backoff, checksum validation, revision deduplication, and raw persistence.| |VEXER-CONN-ORACLE-01-003 – Trust metadata + provenance|Team Vexer Connectors – Oracle|VEXER-CONN-ORACLE-01-002, VEXER-POLICY-01-001|TODO – Emit Oracle signing metadata (PGP/cosign) and provenance hints for consensus weighting.| diff --git a/src/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests/Connectors/RedHatCsafConnectorTests.cs b/src/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests/Connectors/RedHatCsafConnectorTests.cs new file mode 100644 index 00000000..22765fd2 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests/Connectors/RedHatCsafConnectorTests.cs @@ -0,0 +1,277 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Net; +using System.Net.Http; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Vexer.Connectors.Abstractions; +using StellaOps.Vexer.Connectors.RedHat.CSAF.Configuration; +using StellaOps.Vexer.Connectors.RedHat.CSAF.Metadata; +using StellaOps.Vexer.Core; +using StellaOps.Vexer.Storage.Mongo; + +namespace StellaOps.Vexer.Connectors.RedHat.CSAF.Tests.Connectors; + +public sealed class RedHatCsafConnectorTests +{ + private static readonly VexConnectorDescriptor Descriptor = new("vexer:redhat", VexProviderKind.Distro, "Red Hat CSAF"); + + [Fact] + public async Task FetchAsync_EmitsDocumentsAfterSince() + { + var metadata = """ + { + "metadata": { + "provider": { "name": "Red Hat Product Security" } + }, + "distributions": [ + { "directory": "https://example.com/security/data/csaf/v2/advisories/" } + ], + "rolie": { + "feeds": [ + { "url": "https://example.com/security/data/csaf/v2/advisories/rolie/feed.atom" } + ] + } + } + """; + + var feed = """ + + + urn:redhat:1 + 2025-10-16T10:00:00Z + + + + urn:redhat:2 + 2025-10-17T10:00:00Z + + + + """; + + var handler = TestHttpMessageHandler.Create( + request => Response(HttpStatusCode.OK, metadata, "application/json"), + request => Response(HttpStatusCode.OK, feed, "application/atom+xml"), + request => Response(HttpStatusCode.OK, "{ \"csaf\": 1 }", "application/json")); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://example.com/"), + }; + + var factory = new SingleClientHttpClientFactory(httpClient); + var cache = new MemoryCache(new MemoryCacheOptions()); + var options = Options.Create(new RedHatConnectorOptions()); + var metadataLoader = new RedHatProviderMetadataLoader(factory, cache, options, NullLogger.Instance); + var stateRepository = new InMemoryConnectorStateRepository(); + var connector = new RedHatCsafConnector(Descriptor, metadataLoader, factory, stateRepository, NullLogger.Instance, TimeProvider.System); + + var rawSink = new CapturingRawSink(); + var context = new VexConnectorContext( + new DateTimeOffset(2025, 10, 16, 12, 0, 0, TimeSpan.Zero), + VexConnectorSettings.Empty, + rawSink, + new NoopSignatureVerifier(), + new NoopNormalizerRouter(), + new ServiceCollection().BuildServiceProvider()); + + var results = new List(); + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + results.Add(document); + } + + Assert.Single(results); + Assert.Single(rawSink.Documents); + Assert.Equal("https://example.com/doc2.json", results[0].SourceUri.ToString()); + Assert.Equal("https://example.com/doc2.json", rawSink.Documents[0].SourceUri.ToString()); + Assert.Equal(3, handler.CallCount); + stateRepository.State.Should().NotBeNull(); + stateRepository.State!.LastUpdated.Should().Be(new DateTimeOffset(2025, 10, 17, 10, 0, 0, TimeSpan.Zero)); + stateRepository.State.DocumentDigests.Should().HaveCount(1); + } + + [Fact] + public async Task FetchAsync_UsesStateToSkipDuplicateDocuments() + { + var metadata = """ + { + "metadata": { + "provider": { "name": "Red Hat Product Security" } + }, + "distributions": [ + { "directory": "https://example.com/security/data/csaf/v2/advisories/" } + ], + "rolie": { + "feeds": [ + { "url": "https://example.com/security/data/csaf/v2/advisories/rolie/feed.atom" } + ] + } + } + """; + + var feed = """ + + + urn:redhat:1 + 2025-10-17T10:00:00Z + + + + """; + + var handler1 = TestHttpMessageHandler.Create( + _ => Response(HttpStatusCode.OK, metadata, "application/json"), + _ => Response(HttpStatusCode.OK, feed, "application/atom+xml"), + _ => Response(HttpStatusCode.OK, "{ \"csaf\": 1 }", "application/json")); + + var stateRepository = new InMemoryConnectorStateRepository(); + await ExecuteFetchAsync(handler1, stateRepository); + + stateRepository.State.Should().NotBeNull(); + var previousState = stateRepository.State!; + + var handler2 = TestHttpMessageHandler.Create( + _ => Response(HttpStatusCode.OK, metadata, "application/json"), + _ => Response(HttpStatusCode.OK, feed, "application/atom+xml"), + _ => Response(HttpStatusCode.OK, "{ \"csaf\": 1 }", "application/json")); + + var (results, rawSink) = await ExecuteFetchAsync(handler2, stateRepository); + + results.Should().BeEmpty(); + rawSink.Documents.Should().BeEmpty(); + stateRepository.State!.DocumentDigests.Should().Equal(previousState.DocumentDigests); + } + + private static HttpResponseMessage Response(HttpStatusCode statusCode, string content, string contentType) + => new(statusCode) + { + Content = new StringContent(content, Encoding.UTF8, contentType), + }; + + private sealed class CapturingRawSink : IVexRawDocumentSink + { + public List Documents { get; } = new(); + + public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) + { + Documents.Add(document); + return ValueTask.CompletedTask; + } + } + + private sealed class NoopSignatureVerifier : IVexSignatureVerifier + { + public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(null); + } + + private sealed class NoopNormalizerRouter : IVexNormalizerRouter + { + public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); + } + + private sealed class SingleClientHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleClientHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class TestHttpMessageHandler : HttpMessageHandler + { + private readonly Queue> _responders; + + private TestHttpMessageHandler(IEnumerable> responders) + { + _responders = new Queue>(responders); + } + + public int CallCount { get; private set; } + + public static TestHttpMessageHandler Create(params Func[] responders) + => new(responders); + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + CallCount++; + if (_responders.Count == 0) + { + throw new InvalidOperationException("No responder configured for request."); + } + + var responder = _responders.Count > 1 + ? _responders.Dequeue() + : _responders.Peek(); + + var response = responder(request); + response.RequestMessage = request; + return Task.FromResult(response); + } + } + + private static async Task<(List Documents, CapturingRawSink Sink)> ExecuteFetchAsync( + TestHttpMessageHandler handler, + InMemoryConnectorStateRepository stateRepository) + { + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://example.com/"), + }; + + var factory = new SingleClientHttpClientFactory(httpClient); + var cache = new MemoryCache(new MemoryCacheOptions()); + var options = Options.Create(new RedHatConnectorOptions()); + var metadataLoader = new RedHatProviderMetadataLoader(factory, cache, options, NullLogger.Instance); + var connector = new RedHatCsafConnector(Descriptor, metadataLoader, factory, stateRepository, NullLogger.Instance, TimeProvider.System); + + var rawSink = new CapturingRawSink(); + var context = new VexConnectorContext( + null, + VexConnectorSettings.Empty, + rawSink, + new NoopSignatureVerifier(), + new NoopNormalizerRouter(), + new ServiceCollection().BuildServiceProvider()); + + var documents = new List(); + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(document); + } + + return (documents, rawSink); + } + + private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository + { + public VexConnectorState? State { get; private set; } + + public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken) + { + if (State is not null && string.Equals(State.ConnectorId, connectorId, StringComparison.OrdinalIgnoreCase)) + { + return ValueTask.FromResult(State); + } + + return ValueTask.FromResult(null); + } + + public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken) + { + State = state; + return ValueTask.CompletedTask; + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests/Metadata/RedHatProviderMetadataLoaderTests.cs b/src/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests/Metadata/RedHatProviderMetadataLoaderTests.cs new file mode 100644 index 00000000..95d1dec9 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests/Metadata/RedHatProviderMetadataLoaderTests.cs @@ -0,0 +1,235 @@ +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Vexer.Connectors.RedHat.CSAF.Configuration; +using StellaOps.Vexer.Connectors.RedHat.CSAF.Metadata; +using System.IO.Abstractions.TestingHelpers; + +namespace StellaOps.Vexer.Connectors.RedHat.CSAF.Tests.Metadata; + +public sealed class RedHatProviderMetadataLoaderTests +{ + private const string SampleJson = """ + { + "metadata": { + "provider": { + "name": "Red Hat Product Security" + } + }, + "distributions": [ + { "directory": "https://access.redhat.com/security/data/csaf/v2/advisories/" } + ], + "rolie": { + "feeds": [ + { "url": "https://access.redhat.com/security/data/csaf/v2/advisories/rolie/feed.atom" } + ] + } + } + """; + + [Fact] + public async Task LoadAsync_FetchesMetadataAndCaches() + { + var handler = TestHttpMessageHandler.RespondWith(_ => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(SampleJson, Encoding.UTF8, "application/json"), + }; + response.Headers.ETag = new EntityTagHeaderValue("\"abc\""); + return response; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://access.redhat.com/"), + }; + + var factory = new SingleClientHttpClientFactory(httpClient); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var options = new RedHatConnectorOptions + { + MetadataUri = new Uri("https://access.redhat.com/.well-known/csaf/provider-metadata.json"), + OfflineSnapshotPath = fileSystem.Path.Combine("/offline", "redhat-provider.json"), + CosignIssuer = "https://sigstore.dev/redhat", + CosignIdentityPattern = "^spiffe://redhat/.+$", + }; + options.PgpFingerprints.Add("A1B2C3D4E5F6"); + options.Validate(fileSystem); + + var loader = new RedHatProviderMetadataLoader(factory, cache, Options.Create(options), NullLogger.Instance, fileSystem); + + var result = await loader.LoadAsync(CancellationToken.None); + + Assert.Equal("Red Hat Product Security", result.Provider.DisplayName); + Assert.False(result.FromCache); + Assert.False(result.FromOfflineSnapshot); + Assert.Single(result.Provider.BaseUris); + Assert.Equal("https://access.redhat.com/security/data/csaf/v2/advisories/", result.Provider.BaseUris[0].ToString()); + Assert.Equal("https://access.redhat.com/.well-known/csaf/provider-metadata.json", result.Provider.Discovery.WellKnownMetadata?.ToString()); + Assert.Equal("https://access.redhat.com/security/data/csaf/v2/advisories/rolie/feed.atom", result.Provider.Discovery.RolIeService?.ToString()); + Assert.Equal(1.0, result.Provider.Trust.Weight); + Assert.NotNull(result.Provider.Trust.Cosign); + Assert.Equal("https://sigstore.dev/redhat", result.Provider.Trust.Cosign!.Issuer); + Assert.Equal("^spiffe://redhat/.+$", result.Provider.Trust.Cosign.IdentityPattern); + Assert.Contains("A1B2C3D4E5F6", result.Provider.Trust.PgpFingerprints); + Assert.True(fileSystem.FileExists(options.OfflineSnapshotPath)); + Assert.Equal(1, handler.CallCount); + + var second = await loader.LoadAsync(CancellationToken.None); + Assert.True(second.FromCache); + Assert.False(second.FromOfflineSnapshot); + Assert.Equal(1, handler.CallCount); + } + + [Fact] + public async Task LoadAsync_UsesOfflineSnapshotWhenPreferred() + { + var handler = TestHttpMessageHandler.RespondWith(_ => throw new InvalidOperationException("HTTP should not be called")); + + var httpClient = new HttpClient(handler); + var factory = new SingleClientHttpClientFactory(httpClient); + var cache = new MemoryCache(new MemoryCacheOptions()); + + var fileSystem = new MockFileSystem(new Dictionary + { + ["/snapshots/redhat.json"] = new MockFileData(SampleJson), + }); + + var options = new RedHatConnectorOptions + { + MetadataUri = new Uri("https://access.redhat.com/.well-known/csaf/provider-metadata.json"), + OfflineSnapshotPath = "/snapshots/redhat.json", + PreferOfflineSnapshot = true, + PersistOfflineSnapshot = false, + }; + options.Validate(fileSystem); + + var loader = new RedHatProviderMetadataLoader(factory, cache, Options.Create(options), NullLogger.Instance, fileSystem); + + var result = await loader.LoadAsync(CancellationToken.None); + + Assert.Equal("Red Hat Product Security", result.Provider.DisplayName); + Assert.False(result.FromCache); + Assert.True(result.FromOfflineSnapshot); + Assert.Equal(0, handler.CallCount); + + var second = await loader.LoadAsync(CancellationToken.None); + Assert.True(second.FromCache); + Assert.True(second.FromOfflineSnapshot); + Assert.Equal(0, handler.CallCount); + } + + [Fact] + public async Task LoadAsync_UsesETagForConditionalRequest() + { + var handler = TestHttpMessageHandler.Create( + _ => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(SampleJson, Encoding.UTF8, "application/json"), + }; + response.Headers.ETag = new EntityTagHeaderValue("\"abc\""); + return response; + }, + request => + { + Assert.Contains(request.Headers.IfNoneMatch, etag => etag.Tag == "\"abc\""); + return new HttpResponseMessage(HttpStatusCode.NotModified); + }); + + var httpClient = new HttpClient(handler); + var factory = new SingleClientHttpClientFactory(httpClient); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var options = new RedHatConnectorOptions + { + MetadataUri = new Uri("https://access.redhat.com/.well-known/csaf/provider-metadata.json"), + OfflineSnapshotPath = "/offline/redhat.json", + MetadataCacheDuration = TimeSpan.FromMinutes(1), + }; + options.Validate(fileSystem); + + var loader = new RedHatProviderMetadataLoader(factory, cache, Options.Create(options), NullLogger.Instance, fileSystem); + + var first = await loader.LoadAsync(CancellationToken.None); + Assert.False(first.FromCache); + Assert.False(first.FromOfflineSnapshot); + + Assert.True(cache.TryGetValue(RedHatProviderMetadataLoader.CacheKey, out var entryObj)); + Assert.NotNull(entryObj); + + var entryType = entryObj!.GetType(); + var provider = entryType.GetProperty("Provider")!.GetValue(entryObj); + var fetchedAt = entryType.GetProperty("FetchedAt")!.GetValue(entryObj); + var etag = entryType.GetProperty("ETag")!.GetValue(entryObj); + var fromOffline = entryType.GetProperty("FromOffline")!.GetValue(entryObj); + + var expiredEntry = Activator.CreateInstance(entryType, provider, fetchedAt, DateTimeOffset.UtcNow - TimeSpan.FromSeconds(1), etag, fromOffline); + cache.Set(RedHatProviderMetadataLoader.CacheKey, expiredEntry!, new MemoryCacheEntryOptions + { + AbsoluteExpiration = DateTimeOffset.UtcNow + TimeSpan.FromMinutes(1), + }); + + var second = await loader.LoadAsync(CancellationToken.None); + + var third = await loader.LoadAsync(CancellationToken.None); + Assert.True(third.FromCache); + + Assert.Equal(2, handler.CallCount); + } + + private sealed class SingleClientHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleClientHttpClientFactory(HttpClient client) + { + _client = client ?? throw new ArgumentNullException(nameof(client)); + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class TestHttpMessageHandler : HttpMessageHandler + { + private readonly Queue> _responders; + + private TestHttpMessageHandler(IEnumerable> responders) + { + _responders = new Queue>(responders); + } + + public int CallCount { get; private set; } + + public static TestHttpMessageHandler RespondWith(Func responder) + => new(new[] { responder }); + + public static TestHttpMessageHandler Create(params Func[] responders) + => new(responders); + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + CallCount++; + if (_responders.Count == 0) + { + throw new InvalidOperationException("No responder configured for request."); + } + + var responder = _responders.Count > 1 + ? _responders.Dequeue() + : _responders.Peek(); + + var response = responder(request); + response.RequestMessage = request; + return Task.FromResult(response); + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests.csproj b/src/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests.csproj new file mode 100644 index 00000000..e7a249c3 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests.csproj @@ -0,0 +1,17 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/AGENTS.md b/src/StellaOps.Vexer.Connectors.RedHat.CSAF/AGENTS.md index bd139fa0..5c2528d0 100644 --- a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/AGENTS.md +++ b/src/StellaOps.Vexer.Connectors.RedHat.CSAF/AGENTS.md @@ -3,6 +3,8 @@ Connector for Red Hat CSAF VEX feeds, fetching provider metadata, CSAF documents, and projecting them into raw storage for normalization. ## Scope - Discovery via `/.well-known/csaf/provider-metadata.json`, scheduling windows, and ETag-aware HTTP fetches. +- `RedHatProviderMetadataLoader` handles `.well-known` metadata with caching, schema validation, and offline snapshots. +- `RedHatCsafConnector` consumes ROLIE feeds to fetch incremental CSAF documents, honours `context.Since`, and streams raw advisories to storage. - Mapping Red Hat CSAF specifics (product tree aliases, RHSA identifiers, revision history) into raw documents. - Emitting structured telemetry and resume markers for incremental pulls. - Supplying Red Hat-specific trust overrides and provenance hints to normalization. diff --git a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/Configuration/RedHatConnectorOptions.cs b/src/StellaOps.Vexer.Connectors.RedHat.CSAF/Configuration/RedHatConnectorOptions.cs new file mode 100644 index 00000000..9d9bf1ab --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.RedHat.CSAF/Configuration/RedHatConnectorOptions.cs @@ -0,0 +1,104 @@ +using System.Collections.Generic; +using System.IO.Abstractions; + +namespace StellaOps.Vexer.Connectors.RedHat.CSAF.Configuration; + +public sealed class RedHatConnectorOptions +{ + public static readonly Uri DefaultMetadataUri = new("https://access.redhat.com/.well-known/csaf/provider-metadata.json"); + + /// + /// HTTP client name registered for the connector. + /// + public const string HttpClientName = "vexer.connector.redhat"; + + /// + /// URI of the CSAF provider metadata document. + /// + public Uri MetadataUri { get; set; } = DefaultMetadataUri; + + /// + /// Duration to cache loaded metadata before refreshing. + /// + public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromHours(1); + + /// + /// Optional file path used to store or source offline metadata snapshots. + /// + public string? OfflineSnapshotPath { get; set; } + + /// + /// When true, the loader prefers the offline snapshot without attempting a network fetch. + /// + public bool PreferOfflineSnapshot { get; set; } + + /// + /// Enables writing fresh metadata responses to . + /// + public bool PersistOfflineSnapshot { get; set; } = true; + + /// + /// Explicit trust weight override applied to the provider entry. + /// + public double TrustWeight { get; set; } = 1.0; + + /// + /// Sigstore/Cosign issuer used to verify CSAF signatures, if published. + /// + public string? CosignIssuer { get; set; } = "https://access.redhat.com"; + + /// + /// Identity pattern matched against the Cosign certificate subject. + /// + public string? CosignIdentityPattern { get; set; } = "^https://access\\.redhat\\.com/.+$"; + + /// + /// Optional list of PGP fingerprints recognised for Red Hat CSAF artifacts. + /// + public IList PgpFingerprints { get; } = new List(); + + public void Validate(IFileSystem? fileSystem = null) + { + if (MetadataUri is null || !MetadataUri.IsAbsoluteUri) + { + throw new InvalidOperationException("Metadata URI must be absolute."); + } + + if (MetadataUri.Scheme is not ("http" or "https")) + { + throw new InvalidOperationException("Metadata URI must use HTTP or HTTPS."); + } + + if (MetadataCacheDuration <= TimeSpan.Zero) + { + throw new InvalidOperationException("Metadata cache duration must be positive."); + } + + if (!string.IsNullOrWhiteSpace(OfflineSnapshotPath)) + { + var fs = fileSystem ?? new FileSystem(); + var directory = Path.GetDirectoryName(OfflineSnapshotPath); + if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory)) + { + fs.Directory.CreateDirectory(directory); + } + } + + if (double.IsNaN(TrustWeight) || double.IsInfinity(TrustWeight) || TrustWeight <= 0) + { + TrustWeight = 1.0; + } + else if (TrustWeight > 1.0) + { + TrustWeight = 1.0; + } + + if (CosignIssuer is not null) + { + if (string.IsNullOrWhiteSpace(CosignIdentityPattern)) + { + throw new InvalidOperationException("CosignIdentityPattern must be provided when CosignIssuer is specified."); + } + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/DependencyInjection/RedHatConnectorServiceCollectionExtensions.cs b/src/StellaOps.Vexer.Connectors.RedHat.CSAF/DependencyInjection/RedHatConnectorServiceCollectionExtensions.cs new file mode 100644 index 00000000..ed46bc70 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.RedHat.CSAF/DependencyInjection/RedHatConnectorServiceCollectionExtensions.cs @@ -0,0 +1,45 @@ +using System.Net; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using StellaOps.Vexer.Connectors.RedHat.CSAF.Configuration; +using StellaOps.Vexer.Connectors.RedHat.CSAF.Metadata; +using StellaOps.Vexer.Core; +using StellaOps.Vexer.Storage.Mongo; +using System.IO.Abstractions; + +namespace StellaOps.Vexer.Connectors.RedHat.CSAF.DependencyInjection; + +public static class RedHatConnectorServiceCollectionExtensions +{ + public static IServiceCollection AddRedHatCsafConnector(this IServiceCollection services, Action? configure = null) + { + ArgumentNullException.ThrowIfNull(services); + + services.AddOptions() + .Configure(options => + { + configure?.Invoke(options); + }) + .PostConfigure(options => options.Validate()); + + services.TryAddSingleton(); + services.TryAddSingleton(); + + services.AddHttpClient(RedHatConnectorOptions.HttpClientName, client => + { + client.Timeout = TimeSpan.FromSeconds(30); + client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Vexer.Connectors.RedHat/1.0"); + client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); + }) + .ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.All, + }); + + services.AddSingleton(); + services.AddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/Metadata/RedHatProviderMetadataLoader.cs b/src/StellaOps.Vexer.Connectors.RedHat.CSAF/Metadata/RedHatProviderMetadataLoader.cs new file mode 100644 index 00000000..8a38db61 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.RedHat.CSAF/Metadata/RedHatProviderMetadataLoader.cs @@ -0,0 +1,312 @@ +using System.Collections.Immutable; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Vexer.Connectors.RedHat.CSAF.Configuration; +using StellaOps.Vexer.Core; +using System.IO.Abstractions; + +namespace StellaOps.Vexer.Connectors.RedHat.CSAF.Metadata; + +public sealed class RedHatProviderMetadataLoader +{ + public const string CacheKey = "StellaOps.Vexer.Connectors.RedHat.CSAF.Metadata"; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly IMemoryCache _cache; + private readonly ILogger _logger; + private readonly RedHatConnectorOptions _options; + private readonly IFileSystem _fileSystem; + private readonly JsonSerializerOptions _serializerOptions; + private readonly SemaphoreSlim _refreshSemaphore = new(1, 1); + + public RedHatProviderMetadataLoader( + IHttpClientFactory httpClientFactory, + IMemoryCache memoryCache, + IOptions options, + ILogger logger, + IFileSystem? fileSystem = null) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _cache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _fileSystem = fileSystem ?? new FileSystem(); + _serializerOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + }; + } + + public async Task LoadAsync(CancellationToken cancellationToken) + { + if (_cache.TryGetValue(CacheKey, out var cached) && cached is { } cachedEntry && !cachedEntry.IsExpired()) + { + _logger.LogDebug("Returning cached Red Hat provider metadata (expires {Expires}).", cachedEntry.ExpiresAt); + return new RedHatProviderMetadataResult(cachedEntry.Provider, cachedEntry.FetchedAt, true, cachedEntry.FromOffline); + } + + await _refreshSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_cache.TryGetValue(CacheKey, out cached) && cached is { } cachedAfterLock && !cachedAfterLock.IsExpired()) + { + return new RedHatProviderMetadataResult(cachedAfterLock.Provider, cachedAfterLock.FetchedAt, true, cachedAfterLock.FromOffline); + } + + CacheEntry? previous = cached; + + // Attempt live fetch unless offline preferred. + if (!_options.PreferOfflineSnapshot) + { + var httpResult = await TryFetchFromNetworkAsync(previous, cancellationToken).ConfigureAwait(false); + if (httpResult is not null) + { + StoreCache(httpResult); + return new RedHatProviderMetadataResult(httpResult.Provider, httpResult.FetchedAt, false, false); + } + } + + var offlineResult = TryLoadFromOffline(); + if (offlineResult is not null) + { + var offlineEntry = offlineResult with + { + FetchedAt = DateTimeOffset.UtcNow, + ExpiresAt = DateTimeOffset.UtcNow + _options.MetadataCacheDuration, + FromOffline = true, + }; + StoreCache(offlineEntry); + return new RedHatProviderMetadataResult(offlineEntry.Provider, offlineEntry.FetchedAt, false, true); + } + + throw new InvalidOperationException("Unable to load Red Hat CSAF provider metadata from network or offline snapshot."); + } + finally + { + _refreshSemaphore.Release(); + } + } + + private void StoreCache(CacheEntry entry) + { + var cacheEntryOptions = new MemoryCacheEntryOptions + { + AbsoluteExpiration = entry.ExpiresAt, + }; + _cache.Set(CacheKey, entry, cacheEntryOptions); + } + + private async Task TryFetchFromNetworkAsync(CacheEntry? previous, CancellationToken cancellationToken) + { + try + { + var client = _httpClientFactory.CreateClient(RedHatConnectorOptions.HttpClientName); + using var request = new HttpRequestMessage(HttpMethod.Get, _options.MetadataUri); + if (!string.IsNullOrWhiteSpace(previous?.ETag)) + { + if (EntityTagHeaderValue.TryParse(previous.ETag, out var etag)) + { + request.Headers.IfNoneMatch.Add(etag); + } + } + + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + if (response.StatusCode == HttpStatusCode.NotModified && previous is not null) + { + _logger.LogDebug("Red Hat provider metadata not modified (etag {ETag}).", previous.ETag); + return previous with + { + FetchedAt = DateTimeOffset.UtcNow, + ExpiresAt = DateTimeOffset.UtcNow + _options.MetadataCacheDuration, + }; + } + + response.EnsureSuccessStatusCode(); + var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + + var provider = ParseAndValidate(payload); + var etagHeader = response.Headers.ETag?.ToString(); + + if (_options.PersistOfflineSnapshot && !string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath)) + { + try + { + _fileSystem.File.WriteAllText(_options.OfflineSnapshotPath, payload); + _logger.LogDebug("Persisted Red Hat metadata snapshot to {Path}.", _options.OfflineSnapshotPath); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to persist Red Hat metadata snapshot to {Path}.", _options.OfflineSnapshotPath); + } + } + + return new CacheEntry( + provider, + DateTimeOffset.UtcNow, + DateTimeOffset.UtcNow + _options.MetadataCacheDuration, + etagHeader, + FromOffline: false); + } + catch (Exception ex) when (ex is not OperationCanceledException && !_options.PreferOfflineSnapshot) + { + _logger.LogWarning(ex, "Failed to fetch Red Hat provider metadata from {Uri}, will attempt offline snapshot.", _options.MetadataUri); + return null; + } + } + + private CacheEntry? TryLoadFromOffline() + { + if (string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath)) + { + return null; + } + + if (!_fileSystem.File.Exists(_options.OfflineSnapshotPath)) + { + _logger.LogWarning("Offline snapshot path {Path} does not exist.", _options.OfflineSnapshotPath); + return null; + } + + try + { + var payload = _fileSystem.File.ReadAllText(_options.OfflineSnapshotPath); + var provider = ParseAndValidate(payload); + return new CacheEntry(provider, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow + _options.MetadataCacheDuration, ETag: null, FromOffline: true); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to load Red Hat provider metadata from offline snapshot {Path}.", _options.OfflineSnapshotPath); + return null; + } + } + + private VexProvider ParseAndValidate(string payload) + { + if (string.IsNullOrWhiteSpace(payload)) + { + throw new InvalidOperationException("Provider metadata payload was empty."); + } + + ProviderMetadataDocument? document; + try + { + document = JsonSerializer.Deserialize(payload, _serializerOptions); + } + catch (JsonException ex) + { + throw new InvalidOperationException("Provider metadata payload could not be parsed.", ex); + } + + if (document is null) + { + throw new InvalidOperationException("Provider metadata payload was null after parsing."); + } + + if (document.Metadata?.Provider?.Name is null) + { + throw new InvalidOperationException("Provider metadata missing provider name."); + } + + var distributions = document.Distributions? + .Select(static d => d.Directory) + .Where(static s => !string.IsNullOrWhiteSpace(s)) + .Select(static s => CreateUri(s!, nameof(ProviderMetadataDistribution.Directory))) + .ToImmutableArray() ?? ImmutableArray.Empty; + + if (distributions.IsDefaultOrEmpty) + { + throw new InvalidOperationException("Provider metadata did not include any valid distribution directories."); + } + + Uri? rolieFeed = null; + if (document.Rolie?.Feeds is not null) + { + foreach (var feed in document.Rolie.Feeds) + { + if (!string.IsNullOrWhiteSpace(feed.Url)) + { + rolieFeed = CreateUri(feed.Url, "rolie.feeds[].url"); + break; + } + } + } + + var trust = BuildTrust(); + return new VexProvider( + id: "vexer:redhat", + displayName: document.Metadata.Provider.Name, + kind: VexProviderKind.Distro, + baseUris: distributions, + discovery: new VexProviderDiscovery(_options.MetadataUri, rolieFeed), + trust: trust); + } + + private VexProviderTrust BuildTrust() + { + VexCosignTrust? cosign = null; + if (!string.IsNullOrWhiteSpace(_options.CosignIssuer) && !string.IsNullOrWhiteSpace(_options.CosignIdentityPattern)) + { + cosign = new VexCosignTrust(_options.CosignIssuer!, _options.CosignIdentityPattern!); + } + + return new VexProviderTrust( + _options.TrustWeight, + cosign, + _options.PgpFingerprints); + } + + private static Uri CreateUri(string value, string propertyName) + { + if (!Uri.TryCreate(value, UriKind.Absolute, out var uri) || uri.Scheme is not ("http" or "https")) + { + throw new InvalidOperationException($"Provider metadata field '{propertyName}' must be an absolute HTTP(S) URI."); + } + + return uri; + } + + private sealed record ProviderMetadataDocument( + [property: JsonPropertyName("metadata")] ProviderMetadata? Metadata, + [property: JsonPropertyName("distributions")] IReadOnlyList? Distributions, + [property: JsonPropertyName("rolie")] ProviderMetadataRolie? Rolie); + + private sealed record ProviderMetadata( + [property: JsonPropertyName("provider")] ProviderMetadataProvider? Provider); + + private sealed record ProviderMetadataProvider( + [property: JsonPropertyName("name")] string? Name); + + private sealed record ProviderMetadataDistribution( + [property: JsonPropertyName("directory")] string? Directory); + + private sealed record ProviderMetadataRolie( + [property: JsonPropertyName("feeds")] IReadOnlyList? Feeds); + + private sealed record ProviderMetadataRolieFeed( + [property: JsonPropertyName("url")] string? Url); + + private sealed record CacheEntry( + VexProvider Provider, + DateTimeOffset FetchedAt, + DateTimeOffset ExpiresAt, + string? ETag, + bool FromOffline) + { + public bool IsExpired() => DateTimeOffset.UtcNow >= ExpiresAt; + } +} + +public sealed record RedHatProviderMetadataResult( + VexProvider Provider, + DateTimeOffset FetchedAt, + bool FromCache, + bool FromOfflineSnapshot); diff --git a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/RedHatCsafConnector.cs b/src/StellaOps.Vexer.Connectors.RedHat.CSAF/RedHatCsafConnector.cs new file mode 100644 index 00000000..b4cca425 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.RedHat.CSAF/RedHatCsafConnector.cs @@ -0,0 +1,186 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Xml.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Vexer.Connectors.Abstractions; +using StellaOps.Vexer.Connectors.RedHat.CSAF.Configuration; +using StellaOps.Vexer.Connectors.RedHat.CSAF.Metadata; +using StellaOps.Vexer.Core; +using StellaOps.Vexer.Storage.Mongo; + +namespace StellaOps.Vexer.Connectors.RedHat.CSAF; + +public sealed class RedHatCsafConnector : VexConnectorBase +{ + private readonly RedHatProviderMetadataLoader _metadataLoader; + private readonly IHttpClientFactory _httpClientFactory; + private readonly IVexConnectorStateRepository _stateRepository; + public RedHatCsafConnector( + VexConnectorDescriptor descriptor, + RedHatProviderMetadataLoader metadataLoader, + IHttpClientFactory httpClientFactory, + IVexConnectorStateRepository stateRepository, + ILogger logger, + TimeProvider timeProvider) + : base(descriptor, logger, timeProvider) + { + _metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader)); + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + } + + public override ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) + { + // No connector-specific settings yet. + return ValueTask.CompletedTask; + } + + public override async IAsyncEnumerable FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + var metadataResult = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false); + if (metadataResult.Provider.Discovery.RolIeService is null) + { + throw new InvalidOperationException("Red Hat provider metadata did not specify a ROLIE feed."); + } + + var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false); + + var sinceTimestamp = context.Since; + if (state?.LastUpdated is { } persisted && (sinceTimestamp is null || persisted > sinceTimestamp)) + { + sinceTimestamp = persisted; + } + + var knownDigests = state?.DocumentDigests ?? ImmutableArray.Empty; + var digestList = new List(knownDigests); + var digestSet = new HashSet(knownDigests, StringComparer.OrdinalIgnoreCase); + var latestUpdated = state?.LastUpdated ?? sinceTimestamp ?? DateTimeOffset.MinValue; + var stateChanged = false; + + foreach (var entry in await FetchRolieEntriesAsync(metadataResult.Provider.Discovery.RolIeService, cancellationToken).ConfigureAwait(false)) + { + if (sinceTimestamp is not null && entry.Updated is DateTimeOffset updated && updated <= sinceTimestamp) + { + continue; + } + + if (entry.DocumentUri is null) + { + Logger.LogDebug("Skipping ROLIE entry {Id} because no document link was provided.", entry.Id); + continue; + } + + var rawDocument = await DownloadCsafDocumentAsync(entry, cancellationToken).ConfigureAwait(false); + + if (!digestSet.Add(rawDocument.Digest)) + { + Logger.LogDebug("Skipping CSAF document {Uri} because digest {Digest} was already processed.", rawDocument.SourceUri, rawDocument.Digest); + continue; + } + + await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false); + digestList.Add(rawDocument.Digest); + stateChanged = true; + + if (entry.Updated is DateTimeOffset entryUpdated && entryUpdated > latestUpdated) + { + latestUpdated = entryUpdated; + } + + yield return rawDocument; + } + + if (stateChanged) + { + var newLastUpdated = latestUpdated == DateTimeOffset.MinValue ? state?.LastUpdated : latestUpdated; + var updatedState = new VexConnectorState( + Descriptor.Id, + newLastUpdated, + digestList.ToImmutableArray()); + + await _stateRepository.SaveAsync(updatedState, cancellationToken).ConfigureAwait(false); + } + } + + public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + { + // This connector relies on format-specific normalizers registered elsewhere. + throw new NotSupportedException("RedHatCsafConnector does not perform in-line normalization; use the CSAF normalizer component."); + } + + private async Task> FetchRolieEntriesAsync(Uri feedUri, CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(RedHatConnectorOptions.HttpClientName); + using var response = await client.GetAsync(feedUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + var document = XDocument.Load(stream); + var ns = document.Root?.Name.Namespace ?? "http://www.w3.org/2005/Atom"; + + var entries = document.Root? + .Elements(ns + "entry") + .Select(e => new RolieEntry( + Id: (string?)e.Element(ns + "id"), + Updated: ParseUpdated((string?)e.Element(ns + "updated")), + DocumentUri: ParseDocumentLink(e, ns))) + .Where(entry => entry.Id is not null && entry.Updated is not null) + .OrderBy(entry => entry.Updated) + .ToList() ?? new List(); + + return entries; + } + + private static DateTimeOffset? ParseUpdated(string? value) + => DateTimeOffset.TryParse(value, out var parsed) ? parsed : null; + + private static Uri? ParseDocumentLink(XElement entry, XNamespace ns) + { + var linkElements = entry.Elements(ns + "link"); + foreach (var link in linkElements) + { + var rel = (string?)link.Attribute("rel"); + var href = (string?)link.Attribute("href"); + if (string.IsNullOrWhiteSpace(href)) + { + continue; + } + + if (rel is null || rel.Equals("enclosure", StringComparison.OrdinalIgnoreCase) || rel.Equals("alternate", StringComparison.OrdinalIgnoreCase)) + { + if (Uri.TryCreate(href, UriKind.Absolute, out var uri)) + { + return uri; + } + } + } + + return null; + } + + private async Task DownloadCsafDocumentAsync(RolieEntry entry, CancellationToken cancellationToken) + { + var documentUri = entry.DocumentUri ?? throw new InvalidOperationException("ROLIE entry missing document URI."); + + var client = _httpClientFactory.CreateClient(RedHatConnectorOptions.HttpClientName); + using var response = await client.GetAsync(documentUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + var contentBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + var metadata = BuildMetadata(builder => builder + .Add("redhat.csaf.entryId", entry.Id) + .Add("redhat.csaf.documentUri", documentUri.ToString()) + .Add("redhat.csaf.updated", entry.Updated?.ToString("O"))); + + return CreateRawDocument(VexDocumentFormat.Csaf, documentUri, contentBytes, metadata); + } + + private sealed record RolieEntry(string? Id, DateTimeOffset? Updated, Uri? DocumentUri); +} diff --git a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/StellaOps.Vexer.Connectors.RedHat.CSAF.csproj b/src/StellaOps.Vexer.Connectors.RedHat.CSAF/StellaOps.Vexer.Connectors.RedHat.CSAF.csproj new file mode 100644 index 00000000..3c27086b --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.RedHat.CSAF/StellaOps.Vexer.Connectors.RedHat.CSAF.csproj @@ -0,0 +1,19 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/TASKS.md b/src/StellaOps.Vexer.Connectors.RedHat.CSAF/TASKS.md index 0dd22adf..04f43ae1 100644 --- a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/TASKS.md +++ b/src/StellaOps.Vexer.Connectors.RedHat.CSAF/TASKS.md @@ -2,6 +2,9 @@ If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and # TASKS | Task | Owner(s) | Depends on | Notes | |---|---|---|---| -|VEXER-CONN-RH-01-001 – Provider metadata discovery|Team Vexer Connectors – Red Hat|VEXER-CONN-ABS-01-001|TODO – Implement `.well-known` metadata loader with caching, schema validation, and offline snapshot support.| -|VEXER-CONN-RH-01-002 – Incremental CSAF pulls|Team Vexer Connectors – Red Hat|VEXER-CONN-RH-01-001, VEXER-STORAGE-01-003|TODO – Fetch CSAF windows with ETag handling, resume tokens, quarantine on schema errors, and persist raw docs.| -|VEXER-CONN-RH-01-003 – Trust metadata emission|Team Vexer Connectors – Red Hat|VEXER-CONN-RH-01-002, VEXER-POLICY-01-001|TODO – Populate provider trust overrides (cosign issuer, identity regex) and provenance hints for policy evaluation/logging.| +|VEXER-CONN-RH-01-001 – Provider metadata discovery|Team Vexer Connectors – Red Hat|VEXER-CONN-ABS-01-001|**DONE (2025-10-17)** – Added `RedHatProviderMetadataLoader` with HTTP/ETag caching, offline snapshot handling, and validation; exposed DI helper + tests covering live, cached, and offline scenarios.| +|VEXER-CONN-RH-01-002 – Incremental CSAF pulls|Team Vexer Connectors – Red Hat|VEXER-CONN-RH-01-001, VEXER-STORAGE-01-003|**DONE (2025-10-17)** – Implemented `RedHatCsafConnector` with ROLIE feed parsing, incremental filtering via `context.Since`, CSAF document download + metadata capture, and persistence through `IVexRawDocumentSink`; tests cover live fetch/cache/offline scenarios with ETag handling.| +|VEXER-CONN-RH-01-003 – Trust metadata emission|Team Vexer Connectors – Red Hat|VEXER-CONN-RH-01-002, VEXER-POLICY-01-001|**DONE (2025-10-17)** – Provider metadata loader now emits trust overrides (weight, cosign issuer/pattern, PGP fingerprints) and the connector surfaces provenance hints for policy/consensus layers.| +|VEXER-CONN-RH-01-004 – Resume state persistence|Team Vexer Connectors – Red Hat|VEXER-CONN-RH-01-002, VEXER-STORAGE-01-003|**DONE (2025-10-17)** – Connector now loads/saves resume state via `IVexConnectorStateRepository`, tracking last update timestamp and recent document digests to avoid duplicate CSAF ingestion; regression covers state persistence and duplicate skips.| +|VEXER-CONN-RH-01-005 – Worker/WebService integration|Team Vexer Connectors – Red Hat|VEXER-CONN-RH-01-002|**DONE (2025-10-17)** – Worker/WebService now call `AddRedHatCsafConnector`, register the connector + state repo, and default worker scheduling adds the `vexer:redhat` provider so background jobs and orchestration can activate the connector without extra wiring.| +|VEXER-CONN-RH-01-006 – CSAF normalization parity tests|Team Vexer Connectors – Red Hat|VEXER-CONN-RH-01-002, VEXER-FMT-CSAF-01-001|**DONE (2025-10-17)** – Added RHSA fixture-driven regression verifying CSAF normalizer retains Red Hat product metadata, tracking fields, and timestamps (`rhsa-sample.json` + `CsafNormalizerTests.NormalizeAsync_PreservesRedHatSpecificMetadata`).| diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests/Authentication/RancherHubTokenProviderTests.cs b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests/Authentication/RancherHubTokenProviderTests.cs new file mode 100644 index 00000000..3ca92c92 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests/Authentication/RancherHubTokenProviderTests.cs @@ -0,0 +1,138 @@ +using System; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Authentication; +using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Configuration; +using Xunit; + +namespace StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests.Authentication; + +public sealed class RancherHubTokenProviderTests +{ + private const string TokenResponse = "{\"access_token\":\"abc123\",\"token_type\":\"Bearer\",\"expires_in\":3600}"; + + [Fact] + public async Task GetAccessTokenAsync_RequestsAndCachesToken() + { + var handler = TestHttpMessageHandler.RespondWith(request => + { + request.Headers.Authorization.Should().NotBeNull(); + request.Content.Should().NotBeNull(); + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(TokenResponse, Encoding.UTF8, "application/json"), + }; + }); + + var client = new HttpClient(handler) + { + BaseAddress = new Uri("https://identity.suse.com"), + }; + + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var provider = new RancherHubTokenProvider(factory, cache, NullLogger.Instance); + + var options = new RancherHubConnectorOptions + { + ClientId = "client", + ClientSecret = "secret", + TokenEndpoint = new Uri("https://identity.suse.com/oauth/token"), + Audience = "https://vexhub.suse.com", + }; + options.Scopes.Clear(); + options.Scopes.Add("hub.read"); + options.Scopes.Add("hub.events"); + + var token = await provider.GetAccessTokenAsync(options, CancellationToken.None); + token.Should().NotBeNull(); + token!.Value.Should().Be("abc123"); + + var cached = await provider.GetAccessTokenAsync(options, CancellationToken.None); + cached.Should().NotBeNull(); + handler.InvocationCount.Should().Be(1); + } + + [Fact] + public async Task GetAccessTokenAsync_ReturnsNullWhenOfflinePreferred() + { + var handler = TestHttpMessageHandler.RespondWith(_ => new HttpResponseMessage(HttpStatusCode.OK)); + var client = new HttpClient(handler) + { + BaseAddress = new Uri("https://identity.suse.com"), + }; + + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var provider = new RancherHubTokenProvider(factory, cache, NullLogger.Instance); + var options = new RancherHubConnectorOptions + { + PreferOfflineSnapshot = true, + ClientId = "client", + ClientSecret = "secret", + TokenEndpoint = new Uri("https://identity.suse.com/oauth/token"), + }; + + var token = await provider.GetAccessTokenAsync(options, CancellationToken.None); + token.Should().BeNull(); + handler.InvocationCount.Should().Be(0); + } + + [Fact] + public async Task GetAccessTokenAsync_ReturnsNullWithoutCredentials() + { + var handler = TestHttpMessageHandler.RespondWith(_ => new HttpResponseMessage(HttpStatusCode.OK)); + var client = new HttpClient(handler) + { + BaseAddress = new Uri("https://identity.suse.com"), + }; + + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var provider = new RancherHubTokenProvider(factory, cache, NullLogger.Instance); + var options = new RancherHubConnectorOptions(); + + var token = await provider.GetAccessTokenAsync(options, CancellationToken.None); + token.Should().BeNull(); + handler.InvocationCount.Should().Be(0); + } + + private sealed class SingleClientHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleClientHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class TestHttpMessageHandler : HttpMessageHandler + { + private readonly Func _responseFactory; + + private TestHttpMessageHandler(Func responseFactory) + { + _responseFactory = responseFactory; + } + + public int InvocationCount { get; private set; } + + public static TestHttpMessageHandler RespondWith(Func responseFactory) + => new(responseFactory); + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + InvocationCount++; + return Task.FromResult(_responseFactory(request)); + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests/Metadata/RancherHubMetadataLoaderTests.cs b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests/Metadata/RancherHubMetadataLoaderTests.cs new file mode 100644 index 00000000..b47da23c --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests/Metadata/RancherHubMetadataLoaderTests.cs @@ -0,0 +1,178 @@ +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Authentication; +using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Configuration; +using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Metadata; +using System.IO.Abstractions.TestingHelpers; +using System.Threading; +using Xunit; + +namespace StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests.Metadata; + +public sealed class RancherHubMetadataLoaderTests +{ + private const string SampleDiscovery = """ + { + "hubId": "vexer:suse.rancher", + "title": "SUSE Rancher VEX Hub", + "subscription": { + "eventsUri": "https://vexhub.suse.com/api/v1/events", + "checkpointUri": "https://vexhub.suse.com/api/v1/checkpoints", + "requiresAuthentication": true, + "channels": ["rke2", "k3s"], + "scopes": ["hub.read", "hub.events"] + }, + "authentication": { + "tokenUri": "https://identity.suse.com/oauth2/token", + "audience": "https://vexhub.suse.com" + }, + "offline": { + "snapshotUri": "https://downloads.suse.com/vexhub/snapshot.json", + "sha256": "deadbeef", + "updated": "2025-10-10T12:00:00Z" + } + } + """; + + [Fact] + public async Task LoadAsync_FetchesAndCachesMetadata() + { + var handler = TestHttpMessageHandler.RespondWith(_ => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(SampleDiscovery, Encoding.UTF8, "application/json"), + }; + response.Headers.ETag = new EntityTagHeaderValue("\"abc\""); + return response; + }); + + var client = new HttpClient(handler) + { + BaseAddress = new Uri("https://vexhub.suse.com"), + }; + + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var offlinePath = fileSystem.Path.Combine(@"C:\offline", "rancher-hub.json"); + var options = new RancherHubConnectorOptions + { + DiscoveryUri = new Uri("https://vexhub.suse.com/.well-known/vex/rancher-hub.json"), + OfflineSnapshotPath = offlinePath, + }; + + var tokenProvider = new RancherHubTokenProvider(factory, cache, NullLogger.Instance); + var loader = new RancherHubMetadataLoader(factory, cache, tokenProvider, fileSystem, NullLogger.Instance); + + var result = await loader.LoadAsync(options, CancellationToken.None); + + result.FromCache.Should().BeFalse(); + result.FromOfflineSnapshot.Should().BeFalse(); + result.Metadata.Provider.DisplayName.Should().Be("SUSE Rancher VEX Hub"); + result.Metadata.Subscription.EventsUri.Should().Be(new Uri("https://vexhub.suse.com/api/v1/events")); + result.Metadata.Authentication.TokenEndpoint.Should().Be(new Uri("https://identity.suse.com/oauth2/token")); + + // Second call should be served from cache (no additional HTTP invocation). + handler.ResetInvocationCount(); + await loader.LoadAsync(options, CancellationToken.None); + handler.InvocationCount.Should().Be(0); + } + + [Fact] + public async Task LoadAsync_UsesOfflineSnapshotWhenNetworkFails() + { + var handler = TestHttpMessageHandler.RespondWith(_ => throw new HttpRequestException("network down")); + var client = new HttpClient(handler) + { + BaseAddress = new Uri("https://vexhub.suse.com"), + }; + + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var offlinePath = fileSystem.Path.Combine(@"C:\offline", "rancher-hub.json"); + fileSystem.AddFile(offlinePath, new MockFileData(SampleDiscovery)); + + var options = new RancherHubConnectorOptions + { + DiscoveryUri = new Uri("https://vexhub.suse.com/.well-known/vex/rancher-hub.json"), + OfflineSnapshotPath = offlinePath, + }; + + var tokenProvider = new RancherHubTokenProvider(factory, cache, NullLogger.Instance); + var loader = new RancherHubMetadataLoader(factory, cache, tokenProvider, fileSystem, NullLogger.Instance); + + var result = await loader.LoadAsync(options, CancellationToken.None); + result.FromOfflineSnapshot.Should().BeTrue(); + result.Metadata.Subscription.RequiresAuthentication.Should().BeTrue(); + result.Metadata.OfflineSnapshot.Should().NotBeNull(); + } + + [Fact] + public async Task LoadAsync_ThrowsWhenOfflinePreferredButMissing() + { + var handler = TestHttpMessageHandler.RespondWith(_ => throw new HttpRequestException("network down")); + var client = new HttpClient(handler) + { + BaseAddress = new Uri("https://vexhub.suse.com"), + }; + + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var options = new RancherHubConnectorOptions + { + DiscoveryUri = new Uri("https://vexhub.suse.com/.well-known/vex/rancher-hub.json"), + OfflineSnapshotPath = "/offline/missing.json", + PreferOfflineSnapshot = true, + }; + + var tokenProvider = new RancherHubTokenProvider(factory, cache, NullLogger.Instance); + var loader = new RancherHubMetadataLoader(factory, cache, tokenProvider, fileSystem, NullLogger.Instance); + + await Assert.ThrowsAsync(() => loader.LoadAsync(options, CancellationToken.None)); + } + + private sealed class SingleClientHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleClientHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class TestHttpMessageHandler : HttpMessageHandler + { + private readonly Func _responseFactory; + + private TestHttpMessageHandler(Func responseFactory) + { + _responseFactory = responseFactory; + } + + public int InvocationCount { get; private set; } + + public static TestHttpMessageHandler RespondWith(Func responseFactory) + => new(responseFactory); + + public void ResetInvocationCount() => InvocationCount = 0; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + InvocationCount++; + return Task.FromResult(_responseFactory(request)); + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests.csproj b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests.csproj new file mode 100644 index 00000000..07cb02f5 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests.csproj @@ -0,0 +1,17 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Authentication/RancherHubTokenProvider.cs b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Authentication/RancherHubTokenProvider.cs new file mode 100644 index 00000000..6b5ff948 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Authentication/RancherHubTokenProvider.cs @@ -0,0 +1,171 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Configuration; + +namespace StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Authentication; + +public sealed class RancherHubTokenProvider +{ + private const string CachePrefix = "StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Token"; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly IMemoryCache _cache; + private readonly ILogger _logger; + private readonly SemaphoreSlim _semaphore = new(1, 1); + + public RancherHubTokenProvider(IHttpClientFactory httpClientFactory, IMemoryCache cache, ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _cache = cache ?? throw new ArgumentNullException(nameof(cache)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask GetAccessTokenAsync(RancherHubConnectorOptions options, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + + if (options.PreferOfflineSnapshot) + { + _logger.LogDebug("Skipping token request because PreferOfflineSnapshot is enabled."); + return null; + } + + var hasCredentials = !string.IsNullOrWhiteSpace(options.ClientId) && + !string.IsNullOrWhiteSpace(options.ClientSecret) && + options.TokenEndpoint is not null; + + if (!hasCredentials) + { + if (!options.AllowAnonymousDiscovery) + { + _logger.LogDebug("No Rancher hub credentials configured; proceeding without Authorization header."); + } + + return null; + } + + var cacheKey = $"{CachePrefix}:{options.ClientId}"; + if (_cache.TryGetValue(cacheKey, out var cachedToken) && cachedToken is not null && !cachedToken.IsExpired()) + { + return cachedToken; + } + + await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_cache.TryGetValue(cacheKey, out cachedToken) && cachedToken is not null && !cachedToken.IsExpired()) + { + return cachedToken; + } + + var token = await RequestTokenAsync(options, cancellationToken).ConfigureAwait(false); + if (token is not null) + { + var lifetime = token.ExpiresAt - DateTimeOffset.UtcNow; + if (lifetime <= TimeSpan.Zero) + { + lifetime = TimeSpan.FromMinutes(5); + } + + var absoluteExpiration = lifetime > TimeSpan.FromSeconds(30) + ? DateTimeOffset.UtcNow + lifetime - TimeSpan.FromSeconds(30) + : DateTimeOffset.UtcNow + TimeSpan.FromSeconds(10); + + _cache.Set(cacheKey, token, new MemoryCacheEntryOptions + { + AbsoluteExpiration = absoluteExpiration, + }); + } + + return token; + } + finally + { + _semaphore.Release(); + } + } + + private async Task RequestTokenAsync(RancherHubConnectorOptions options, CancellationToken cancellationToken) + { + using var request = new HttpRequestMessage(HttpMethod.Post, options.TokenEndpoint); + request.Headers.Accept.ParseAdd("application/json"); + + var parameters = new Dictionary + { + ["grant_type"] = "client_credentials", + }; + + if (options.Scopes.Count > 0) + { + parameters["scope"] = string.Join(' ', options.Scopes); + } + + if (!string.IsNullOrWhiteSpace(options.Audience)) + { + parameters["audience"] = options.Audience!; + } + + if (string.Equals(options.ClientAuthenticationScheme, "client_secret_basic", StringComparison.Ordinal)) + { + var credentials = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{options.ClientId}:{options.ClientSecret}")); + request.Headers.Authorization = new AuthenticationHeaderValue("Basic", credentials); + } + else + { + parameters["client_id"] = options.ClientId!; + parameters["client_secret"] = options.ClientSecret!; + } + + request.Content = new FormUrlEncodedContent(parameters); + + var client = _httpClientFactory.CreateClient(RancherHubConnectorOptions.HttpClientName); + using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException($"Failed to acquire Rancher hub access token ({response.StatusCode}): {payload}"); + } + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); + var root = document.RootElement; + + if (!root.TryGetProperty("access_token", out var accessTokenProperty) || accessTokenProperty.ValueKind is not JsonValueKind.String) + { + throw new InvalidOperationException("Token endpoint response missing access_token."); + } + + var token = accessTokenProperty.GetString(); + if (string.IsNullOrWhiteSpace(token)) + { + throw new InvalidOperationException("Token endpoint response contained an empty access_token."); + } + + var tokenType = root.TryGetProperty("token_type", out var tokenTypeElement) && tokenTypeElement.ValueKind is JsonValueKind.String + ? tokenTypeElement.GetString() ?? "Bearer" + : "Bearer"; + + var expires = root.TryGetProperty("expires_in", out var expiresElement) && + expiresElement.ValueKind is JsonValueKind.Number && + expiresElement.TryGetInt32(out var expiresSeconds) + ? DateTimeOffset.UtcNow + TimeSpan.FromSeconds(Math.Max(30, expiresSeconds)) + : DateTimeOffset.UtcNow + TimeSpan.FromMinutes(30); + + _logger.LogDebug("Acquired Rancher hub access token (expires {Expires}).", expires); + + return new RancherHubAccessToken(token, tokenType, expires); + } +} + +public sealed record RancherHubAccessToken(string Value, string TokenType, DateTimeOffset ExpiresAt) +{ + public bool IsExpired() => DateTimeOffset.UtcNow >= ExpiresAt - TimeSpan.FromMinutes(1); +} diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Configuration/RancherHubConnectorOptions.cs b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Configuration/RancherHubConnectorOptions.cs new file mode 100644 index 00000000..83e0bea1 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Configuration/RancherHubConnectorOptions.cs @@ -0,0 +1,186 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Abstractions; + +namespace StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Configuration; + +public sealed class RancherHubConnectorOptions +{ + public static readonly Uri DefaultDiscoveryUri = new("https://vexhub.suse.com/.well-known/vex/rancher-hub.json"); + + /// + /// HTTP client name registered for the connector. + /// + public const string HttpClientName = "vexer.connector.suse.rancherhub"; + + /// + /// URI for the Rancher VEX hub discovery document. + /// + public Uri DiscoveryUri { get; set; } = DefaultDiscoveryUri; + + /// + /// Optional OAuth2/OIDC token endpoint used for hub authentication. + /// + public Uri? TokenEndpoint { get; set; } + + /// + /// Client identifier used when requesting hub access tokens. + /// + public string? ClientId { get; set; } + + /// + /// Client secret used when requesting hub access tokens. + /// + public string? ClientSecret { get; set; } + + /// + /// OAuth scopes requested for hub access; defaults align with Rancher hub reader role. + /// + public IList Scopes { get; } = new List { "hub.read" }; + + /// + /// Optional audience claim passed when requesting tokens (client credential grant). + /// + public string? Audience { get; set; } + + /// + /// Preferred authentication scheme. Supported: client_secret_basic (default) or client_secret_post. + /// + public string ClientAuthenticationScheme { get; set; } = "client_secret_basic"; + + /// + /// Duration to cache discovery metadata before re-fetching. + /// + public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromMinutes(30); + + /// + /// Optional file path for discovery metadata snapshots. + /// + public string? OfflineSnapshotPath { get; set; } + + /// + /// When true, the loader prefers the offline snapshot prior to attempting network discovery. + /// + public bool PreferOfflineSnapshot { get; set; } + + /// + /// Enables persisting freshly fetched discovery documents to . + /// + public bool PersistOfflineSnapshot { get; set; } = true; + + /// + /// Weight applied to the provider entry; hubs default below direct vendor feeds. + /// + public double TrustWeight { get; set; } = 0.6; + + /// + /// Optional Sigstore/Cosign issuer for verifying hub-delivered attestations. + /// + public string? CosignIssuer { get; set; } + + /// + /// Cosign identity pattern matched against transparency log subjects. + /// + public string? CosignIdentityPattern { get; set; } + + /// + /// Additional trusted PGP fingerprints declared by the hub. + /// + public IList PgpFingerprints { get; } = new List(); + + /// + /// Allows falling back to unauthenticated discovery requests when credentials are absent. + /// + public bool AllowAnonymousDiscovery { get; set; } + + public void Validate(IFileSystem? fileSystem = null) + { + if (DiscoveryUri is null || !DiscoveryUri.IsAbsoluteUri) + { + throw new InvalidOperationException("DiscoveryUri must be an absolute URI."); + } + + if (DiscoveryUri.Scheme is not ("http" or "https")) + { + throw new InvalidOperationException("DiscoveryUri must use HTTP or HTTPS."); + } + + if (MetadataCacheDuration <= TimeSpan.Zero) + { + throw new InvalidOperationException("MetadataCacheDuration must be positive."); + } + + if (!string.IsNullOrWhiteSpace(OfflineSnapshotPath)) + { + var fs = fileSystem ?? new FileSystem(); + var directory = Path.GetDirectoryName(OfflineSnapshotPath); + if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory)) + { + fs.Directory.CreateDirectory(directory); + } + } + + if (PreferOfflineSnapshot && string.IsNullOrWhiteSpace(OfflineSnapshotPath)) + { + throw new InvalidOperationException("OfflineSnapshotPath must be provided when PreferOfflineSnapshot is enabled."); + } + + var hasClientId = !string.IsNullOrWhiteSpace(ClientId); + var hasClientSecret = !string.IsNullOrWhiteSpace(ClientSecret); + var hasTokenEndpoint = TokenEndpoint is not null; + if (hasClientId || hasClientSecret || hasTokenEndpoint) + { + if (!(hasClientId && hasClientSecret && hasTokenEndpoint)) + { + throw new InvalidOperationException("ClientId, ClientSecret, and TokenEndpoint must be provided together for authenticated discovery."); + } + + if (TokenEndpoint is not null && (!TokenEndpoint.IsAbsoluteUri || TokenEndpoint.Scheme is not ("http" or "https"))) + { + throw new InvalidOperationException("TokenEndpoint must be an absolute HTTP(S) URI."); + } + } + + if (double.IsNaN(TrustWeight) || double.IsInfinity(TrustWeight)) + { + TrustWeight = 0.6; + } + else if (TrustWeight <= 0) + { + TrustWeight = 0.1; + } + else if (TrustWeight > 1.0) + { + TrustWeight = 1.0; + } + + if (!string.IsNullOrWhiteSpace(CosignIssuer) && string.IsNullOrWhiteSpace(CosignIdentityPattern)) + { + throw new InvalidOperationException("CosignIdentityPattern must be provided when CosignIssuer is specified."); + } + + if (!string.Equals(ClientAuthenticationScheme, "client_secret_basic", StringComparison.Ordinal) && + !string.Equals(ClientAuthenticationScheme, "client_secret_post", StringComparison.Ordinal)) + { + throw new InvalidOperationException("ClientAuthenticationScheme must be 'client_secret_basic' or 'client_secret_post'."); + } + + // Remove any empty scopes to avoid token request issues. + if (Scopes.Count > 0) + { + for (var i = Scopes.Count - 1; i >= 0; i--) + { + if (string.IsNullOrWhiteSpace(Scopes[i])) + { + Scopes.RemoveAt(i); + } + } + } + + if (Scopes.Count == 0) + { + Scopes.Add("hub.read"); + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Configuration/RancherHubConnectorOptionsValidator.cs b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Configuration/RancherHubConnectorOptionsValidator.cs new file mode 100644 index 00000000..4c29f5c5 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Configuration/RancherHubConnectorOptionsValidator.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using System.IO.Abstractions; +using StellaOps.Vexer.Connectors.Abstractions; + +namespace StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Configuration; + +public sealed class RancherHubConnectorOptionsValidator : IVexConnectorOptionsValidator +{ + private readonly IFileSystem _fileSystem; + + public RancherHubConnectorOptionsValidator(IFileSystem fileSystem) + { + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + } + + public void Validate(VexConnectorDescriptor descriptor, RancherHubConnectorOptions options, IList errors) + { + ArgumentNullException.ThrowIfNull(descriptor); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(errors); + + try + { + options.Validate(_fileSystem); + } + catch (Exception ex) + { + errors.Add(ex.Message); + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/DependencyInjection/RancherHubConnectorServiceCollectionExtensions.cs b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/DependencyInjection/RancherHubConnectorServiceCollectionExtensions.cs new file mode 100644 index 00000000..3e8b23e7 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/DependencyInjection/RancherHubConnectorServiceCollectionExtensions.cs @@ -0,0 +1,49 @@ +using System; +using System.Net; +using System.Net.Http; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using StellaOps.Vexer.Connectors.Abstractions; +using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Authentication; +using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Configuration; +using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Metadata; +using StellaOps.Vexer.Core; +using System.IO.Abstractions; + +namespace StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.DependencyInjection; + +public static class RancherHubConnectorServiceCollectionExtensions +{ + public static IServiceCollection AddRancherHubConnector(this IServiceCollection services, Action? configure = null) + { + ArgumentNullException.ThrowIfNull(services); + + services.TryAddSingleton(); + services.TryAddSingleton(); + + services.AddOptions() + .Configure(options => + { + configure?.Invoke(options); + }); + + services.AddSingleton, RancherHubConnectorOptionsValidator>(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + services.AddHttpClient(RancherHubConnectorOptions.HttpClientName, client => + { + client.Timeout = TimeSpan.FromSeconds(30); + client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/1.0"); + client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); + }) + .ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.All, + }); + + return services; + } +} diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Metadata/RancherHubMetadataLoader.cs b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Metadata/RancherHubMetadataLoader.cs new file mode 100644 index 00000000..930f8192 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Metadata/RancherHubMetadataLoader.cs @@ -0,0 +1,455 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO.Abstractions; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Authentication; +using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Configuration; +using StellaOps.Vexer.Core; + +namespace StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Metadata; + +public sealed class RancherHubMetadataLoader +{ + public const string CachePrefix = "StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Metadata"; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly IMemoryCache _memoryCache; + private readonly RancherHubTokenProvider _tokenProvider; + private readonly IFileSystem _fileSystem; + private readonly ILogger _logger; + private readonly SemaphoreSlim _semaphore = new(1, 1); + private readonly JsonDocumentOptions _documentOptions; + + public RancherHubMetadataLoader( + IHttpClientFactory httpClientFactory, + IMemoryCache memoryCache, + RancherHubTokenProvider tokenProvider, + IFileSystem fileSystem, + ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache)); + _tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider)); + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _documentOptions = new JsonDocumentOptions + { + CommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true, + }; + } + + public async Task LoadAsync(RancherHubConnectorOptions options, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + + var cacheKey = CreateCacheKey(options); + if (_memoryCache.TryGetValue(cacheKey, out var cached) && cached is not null && !cached.IsExpired()) + { + _logger.LogDebug("Returning cached Rancher hub metadata (expires {Expires}).", cached.ExpiresAt); + return new RancherHubMetadataResult(cached.Metadata, cached.FetchedAt, FromCache: true, cached.FromOfflineSnapshot); + } + + await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_memoryCache.TryGetValue(cacheKey, out cached) && cached is not null && !cached.IsExpired()) + { + return new RancherHubMetadataResult(cached.Metadata, cached.FetchedAt, FromCache: true, cached.FromOfflineSnapshot); + } + + CacheEntry? previous = cached; + CacheEntry? entry = null; + + if (options.PreferOfflineSnapshot) + { + entry = TryLoadFromOffline(options); + if (entry is null) + { + throw new InvalidOperationException("PreferOfflineSnapshot is enabled but no offline discovery snapshot was found."); + } + } + else + { + entry = await TryFetchFromNetworkAsync(options, previous, cancellationToken).ConfigureAwait(false); + if (entry is null) + { + entry = TryLoadFromOffline(options); + } + } + + if (entry is null) + { + throw new InvalidOperationException("Unable to load Rancher hub discovery metadata from network or offline snapshot."); + } + + _memoryCache.Set(cacheKey, entry, new MemoryCacheEntryOptions + { + AbsoluteExpirationRelativeToNow = options.MetadataCacheDuration, + }); + + return new RancherHubMetadataResult(entry.Metadata, entry.FetchedAt, FromCache: false, entry.FromOfflineSnapshot); + } + finally + { + _semaphore.Release(); + } + } + + private async Task TryFetchFromNetworkAsync(RancherHubConnectorOptions options, CacheEntry? previous, CancellationToken cancellationToken) + { + try + { + var client = _httpClientFactory.CreateClient(RancherHubConnectorOptions.HttpClientName); + using var request = new HttpRequestMessage(HttpMethod.Get, options.DiscoveryUri); + request.Headers.Accept.ParseAdd("application/json"); + + if (!string.IsNullOrWhiteSpace(previous?.ETag) && EntityTagHeaderValue.TryParse(previous.ETag, out var etag)) + { + request.Headers.IfNoneMatch.Add(etag); + } + + var token = await _tokenProvider.GetAccessTokenAsync(options, cancellationToken).ConfigureAwait(false); + if (token is not null) + { + var scheme = string.IsNullOrWhiteSpace(token.TokenType) ? "Bearer" : token.TokenType; + request.Headers.Authorization = new AuthenticationHeaderValue(scheme, token.Value); + } + + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + if (response.StatusCode == HttpStatusCode.NotModified && previous is not null) + { + _logger.LogDebug("Rancher hub discovery document not modified (etag {ETag}).", previous.ETag); + return previous with + { + FetchedAt = DateTimeOffset.UtcNow, + ExpiresAt = DateTimeOffset.UtcNow + options.MetadataCacheDuration, + FromOfflineSnapshot = false, + }; + } + + response.EnsureSuccessStatusCode(); + var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var metadata = ParseMetadata(payload, options); + var entry = new CacheEntry( + metadata, + DateTimeOffset.UtcNow, + DateTimeOffset.UtcNow + options.MetadataCacheDuration, + response.Headers.ETag?.ToString(), + FromOfflineSnapshot: false, + Payload: payload); + + PersistOfflineSnapshot(options, payload); + return entry; + } + catch (Exception ex) when (ex is not OperationCanceledException && !options.PreferOfflineSnapshot) + { + _logger.LogWarning(ex, "Failed to fetch Rancher hub discovery document; attempting offline snapshot fallback."); + return null; + } + } + + private CacheEntry? TryLoadFromOffline(RancherHubConnectorOptions options) + { + if (string.IsNullOrWhiteSpace(options.OfflineSnapshotPath)) + { + return null; + } + + if (!_fileSystem.File.Exists(options.OfflineSnapshotPath)) + { + _logger.LogWarning("Rancher hub offline snapshot path {Path} does not exist.", options.OfflineSnapshotPath); + return null; + } + + try + { + var payload = _fileSystem.File.ReadAllText(options.OfflineSnapshotPath); + var metadata = ParseMetadata(payload, options); + return new CacheEntry( + metadata, + DateTimeOffset.UtcNow, + DateTimeOffset.UtcNow + options.MetadataCacheDuration, + ETag: null, + FromOfflineSnapshot: true, + Payload: payload); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to load Rancher hub discovery metadata from offline snapshot {Path}.", options.OfflineSnapshotPath); + return null; + } + } + + private void PersistOfflineSnapshot(RancherHubConnectorOptions options, string payload) + { + if (!options.PersistOfflineSnapshot || string.IsNullOrWhiteSpace(options.OfflineSnapshotPath)) + { + return; + } + + try + { + var directory = _fileSystem.Path.GetDirectoryName(options.OfflineSnapshotPath); + if (!string.IsNullOrWhiteSpace(directory)) + { + _fileSystem.Directory.CreateDirectory(directory); + } + + _fileSystem.File.WriteAllText(options.OfflineSnapshotPath, payload); + _logger.LogDebug("Persisted Rancher hub discovery snapshot to {Path}.", options.OfflineSnapshotPath); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to persist Rancher hub discovery snapshot to {Path}.", options.OfflineSnapshotPath); + } + } + + private RancherHubMetadata ParseMetadata(string payload, RancherHubConnectorOptions options) + { + if (string.IsNullOrWhiteSpace(payload)) + { + throw new InvalidOperationException("Rancher hub discovery payload was empty."); + } + + try + { + using var document = JsonDocument.Parse(payload, _documentOptions); + var root = document.RootElement; + + var hubId = ReadString(root, "hubId") ?? "vexer:suse:rancher"; + var title = ReadString(root, "title") ?? ReadString(root, "displayName") ?? "SUSE Rancher VEX Hub"; + var baseUri = ReadUri(root, "baseUri"); + + var subscriptionElement = TryGetProperty(root, "subscription"); + if (!subscriptionElement.HasValue) + { + throw new InvalidOperationException("Discovery payload missing subscription section."); + } + + var subscription = subscriptionElement.Value; + var eventsUri = ReadRequiredUri(subscription, "eventsUri", "eventsUrl", "eventsEndpoint"); + var checkpointUri = ReadUri(subscription, "checkpointUri", "checkpointUrl", "checkpointEndpoint"); + var channels = ReadStringArray(subscription, "channels", "defaultChannels", "products"); + var scopes = ReadStringArray(subscription, "scopes", "defaultScopes"); + var requiresAuth = ReadBoolean(subscription, "requiresAuthentication", defaultValue: options.TokenEndpoint is not null); + + var authenticationElement = TryGetProperty(root, "authentication"); + var tokenEndpointFromMetadata = authenticationElement.HasValue + ? ReadUri(authenticationElement.Value, "tokenUri", "tokenEndpoint") ?? options.TokenEndpoint + : options.TokenEndpoint; + var audience = authenticationElement.HasValue + ? ReadString(authenticationElement.Value, "audience", "aud") ?? options.Audience + : options.Audience; + + var offlineElement = TryGetProperty(root, "offline", "snapshot"); + var offlineSnapshot = offlineElement.HasValue + ? BuildOfflineSnapshot(offlineElement.Value, options) + : null; + + var provider = BuildProvider(hubId, title, baseUri, eventsUri, options); + var subscriptionMetadata = new RancherHubSubscriptionMetadata(eventsUri, checkpointUri, channels, scopes, requiresAuth); + var authenticationMetadata = new RancherHubAuthenticationMetadata(tokenEndpointFromMetadata, audience); + + return new RancherHubMetadata(provider, subscriptionMetadata, authenticationMetadata, offlineSnapshot); + } + catch (JsonException ex) + { + throw new InvalidOperationException("Failed to parse Rancher hub discovery payload.", ex); + } + } + + private RancherHubOfflineSnapshotMetadata? BuildOfflineSnapshot(JsonElement element, RancherHubConnectorOptions options) + { + var snapshotUri = ReadUri(element, "snapshotUri", "uri", "url"); + if (snapshotUri is null) + { + return null; + } + + var checksum = ReadString(element, "sha256", "checksum", "digest"); + DateTimeOffset? updatedAt = null; + var updatedString = ReadString(element, "updated", "lastModified", "timestamp"); + if (!string.IsNullOrWhiteSpace(updatedString) && DateTimeOffset.TryParse(updatedString, out var parsed)) + { + updatedAt = parsed; + } + + return new RancherHubOfflineSnapshotMetadata(snapshotUri, checksum, updatedAt); + } + + private VexProvider BuildProvider(string hubId, string title, Uri? baseUri, Uri eventsUri, RancherHubConnectorOptions options) + { + var baseUris = new List(); + if (baseUri is not null) + { + baseUris.Add(baseUri); + } + baseUris.Add(eventsUri); + + VexCosignTrust? cosign = null; + if (!string.IsNullOrWhiteSpace(options.CosignIssuer) && !string.IsNullOrWhiteSpace(options.CosignIdentityPattern)) + { + cosign = new VexCosignTrust(options.CosignIssuer!, options.CosignIdentityPattern!); + } + + var trust = new VexProviderTrust(options.TrustWeight, cosign, options.PgpFingerprints); + return new VexProvider(hubId, title, VexProviderKind.Hub, baseUris, new VexProviderDiscovery(options.DiscoveryUri, null), trust); + } + + private static string CreateCacheKey(RancherHubConnectorOptions options) + => $"{CachePrefix}:{options.DiscoveryUri}"; + + private static JsonElement? TryGetProperty(JsonElement element, params string[] propertyNames) + { + foreach (var name in propertyNames) + { + if (element.TryGetProperty(name, out var value) && value.ValueKind is not JsonValueKind.Null and not JsonValueKind.Undefined) + { + return value; + } + } + + return null; + } + + private static string? ReadString(JsonElement element, params string[] propertyNames) + { + var property = TryGetProperty(element, propertyNames); + if (property is null || property.Value.ValueKind is not JsonValueKind.String) + { + return null; + } + + var value = property.Value.GetString(); + return string.IsNullOrWhiteSpace(value) ? null : value; + } + + private static bool ReadBoolean(JsonElement element, string propertyName, bool defaultValue) + { + if (!element.TryGetProperty(propertyName, out var property)) + { + return defaultValue; + } + + return property.ValueKind switch + { + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.String when bool.TryParse(property.GetString(), out var parsed) => parsed, + _ => defaultValue, + }; + } + + private static ImmutableArray ReadStringArray(JsonElement element, params string[] propertyNames) + { + var property = TryGetProperty(element, propertyNames); + if (property is null) + { + return ImmutableArray.Empty; + } + + if (property.Value.ValueKind is JsonValueKind.Array) + { + var builder = ImmutableArray.CreateBuilder(); + foreach (var item in property.Value.EnumerateArray()) + { + if (item.ValueKind is JsonValueKind.String) + { + var value = item.GetString(); + if (!string.IsNullOrWhiteSpace(value)) + { + builder.Add(value!); + } + } + } + + return builder.Count == 0 ? ImmutableArray.Empty : builder.ToImmutable(); + } + + if (property.Value.ValueKind is JsonValueKind.String) + { + var single = property.Value.GetString(); + return string.IsNullOrWhiteSpace(single) + ? ImmutableArray.Empty + : ImmutableArray.Create(single!); + } + + return ImmutableArray.Empty; + } + + private static Uri? ReadUri(JsonElement element, params string[] propertyNames) + { + var value = ReadString(element, propertyNames); + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + if (!Uri.TryCreate(value, UriKind.Absolute, out var uri) || uri.Scheme is not ("http" or "https")) + { + throw new InvalidOperationException($"Discovery field '{string.Join("/", propertyNames)}' must be an absolute HTTP(S) URI."); + } + + return uri; + } + + private static Uri ReadRequiredUri(JsonElement element, params string[] propertyNames) + { + var uri = ReadUri(element, propertyNames); + if (uri is null) + { + throw new InvalidOperationException($"Discovery payload missing required URI field '{string.Join("/", propertyNames)}'."); + } + + return uri; + } + + private sealed record CacheEntry( + RancherHubMetadata Metadata, + DateTimeOffset FetchedAt, + DateTimeOffset ExpiresAt, + string? ETag, + bool FromOfflineSnapshot, + string? Payload) + { + public bool IsExpired() => DateTimeOffset.UtcNow >= ExpiresAt; + } +} + +public sealed record RancherHubMetadata( + VexProvider Provider, + RancherHubSubscriptionMetadata Subscription, + RancherHubAuthenticationMetadata Authentication, + RancherHubOfflineSnapshotMetadata? OfflineSnapshot); + +public sealed record RancherHubSubscriptionMetadata( + Uri EventsUri, + Uri? CheckpointUri, + ImmutableArray Channels, + ImmutableArray Scopes, + bool RequiresAuthentication); + +public sealed record RancherHubAuthenticationMetadata( + Uri? TokenEndpoint, + string? Audience); + +public sealed record RancherHubOfflineSnapshotMetadata( + Uri SnapshotUri, + string? Sha256, + DateTimeOffset? UpdatedAt); + +public sealed record RancherHubMetadataResult( + RancherHubMetadata Metadata, + DateTimeOffset FetchedAt, + bool FromCache, + bool FromOfflineSnapshot); diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/RancherHubConnector.cs b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/RancherHubConnector.cs new file mode 100644 index 00000000..4b638586 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/RancherHubConnector.cs @@ -0,0 +1,85 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.Logging; +using StellaOps.Vexer.Connectors.Abstractions; +using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Configuration; +using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Metadata; +using StellaOps.Vexer.Core; + +namespace StellaOps.Vexer.Connectors.SUSE.RancherVEXHub; + +public sealed class RancherHubConnector : VexConnectorBase +{ + private static readonly VexConnectorDescriptor StaticDescriptor = new( + id: "vexer:suse.rancher", + kind: VexProviderKind.Hub, + displayName: "SUSE Rancher VEX Hub") + { + Tags = ImmutableArray.Create("hub", "suse", "offline"), + }; + + private readonly RancherHubMetadataLoader _metadataLoader; + private readonly IEnumerable> _validators; + + private RancherHubConnectorOptions? _options; + private RancherHubMetadataResult? _metadata; + + public RancherHubConnector( + RancherHubMetadataLoader metadataLoader, + ILogger logger, + TimeProvider timeProvider, + IEnumerable>? validators = null) + : base(StaticDescriptor, logger, timeProvider) + { + _metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader)); + _validators = validators ?? Array.Empty>(); + } + + public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) + { + _options = VexConnectorOptionsBinder.Bind( + Descriptor, + settings, + validators: _validators); + + _metadata = await _metadataLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); + + LogConnectorEvent(LogLevel.Information, "validate", "Rancher hub discovery loaded.", new Dictionary + { + ["discoveryUri"] = _options.DiscoveryUri.ToString(), + ["subscriptionUri"] = _metadata.Metadata.Subscription.EventsUri.ToString(), + ["requiresAuth"] = _metadata.Metadata.Subscription.RequiresAuthentication, + ["fromOffline"] = _metadata.FromOfflineSnapshot, + }); + } + + public override async IAsyncEnumerable FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + if (_options is null) + { + throw new InvalidOperationException("Connector must be validated before fetch operations."); + } + + if (_metadata is null) + { + _metadata = await _metadataLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); + } + + LogConnectorEvent(LogLevel.Debug, "fetch", "Rancher hub connector discovery ready; event ingestion will be implemented in VEXER-CONN-SUSE-01-002.", new Dictionary + { + ["since"] = context.Since?.ToString("O"), + ["subscriptionUri"] = _metadata.Metadata.Subscription.EventsUri.ToString(), + }); + + yield break; + } + + public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => throw new NotSupportedException("RancherHubConnector relies on format-specific normalizers for CSAF/OpenVEX payloads."); + + public RancherHubMetadata? GetCachedMetadata() => _metadata?.Metadata; +} diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.csproj b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.csproj new file mode 100644 index 00000000..3c27086b --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.csproj @@ -0,0 +1,19 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/TASKS.md b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/TASKS.md index 4f8cf868..9ce451c6 100644 --- a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/TASKS.md +++ b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/TASKS.md @@ -2,6 +2,6 @@ If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and # TASKS | Task | Owner(s) | Depends on | Notes | |---|---|---|---| -|VEXER-CONN-SUSE-01-001 – Rancher hub discovery & auth|Team Vexer Connectors – SUSE|VEXER-CONN-ABS-01-001|TODO – Implement hub discovery/subscription setup with credential handling and offline snapshot support.| +|VEXER-CONN-SUSE-01-001 – Rancher hub discovery & auth|Team Vexer Connectors – SUSE|VEXER-CONN-ABS-01-001|**DONE (2025-10-17)** – Added Rancher hub options/token provider, discovery metadata loader with offline snapshots + caching, connector shell, DI wiring, and unit tests covering network/offline paths.| |VEXER-CONN-SUSE-01-002 – Checkpointed event ingestion|Team Vexer Connectors – SUSE|VEXER-CONN-SUSE-01-001, VEXER-STORAGE-01-003|TODO – Process hub events with resume checkpoints, deduplication, and quarantine path for malformed payloads.| |VEXER-CONN-SUSE-01-003 – Trust metadata & policy hints|Team Vexer Connectors – SUSE|VEXER-CONN-SUSE-01-002, VEXER-POLICY-01-001|TODO – Emit provider trust configuration (signers, weight overrides) and attach provenance hints for consensus engine.| diff --git a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests/Metadata/UbuntuCatalogLoaderTests.cs b/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests/Metadata/UbuntuCatalogLoaderTests.cs new file mode 100644 index 00000000..363fb91d --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests/Metadata/UbuntuCatalogLoaderTests.cs @@ -0,0 +1,172 @@ +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Vexer.Connectors.Ubuntu.CSAF.Configuration; +using StellaOps.Vexer.Connectors.Ubuntu.CSAF.Metadata; +using System.IO.Abstractions.TestingHelpers; +using Xunit; +using System.Threading; + +namespace StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests.Metadata; + +public sealed class UbuntuCatalogLoaderTests +{ + private const string SampleIndex = """ + { + "generated": "2025-10-10T00:00:00Z", + "channels": [ + { + "name": "stable", + "catalogUrl": "https://ubuntu.com/security/csaf/stable/catalog.json", + "sha256": "abc", + "lastUpdated": "2025-10-09T10:00:00Z" + }, + { + "name": "esm", + "catalogUrl": "https://ubuntu.com/security/csaf/esm/catalog.json", + "sha256": "def", + "lastUpdated": "2025-10-08T10:00:00Z" + } + ] + } + """; + + [Fact] + public async Task LoadAsync_FetchesAndCachesIndex() + { + var handler = new TestHttpMessageHandler(new Dictionary + { + [new Uri("https://ubuntu.com/security/csaf/index.json")] = CreateResponse(SampleIndex), + }); + var client = new HttpClient(handler); + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var loader = new UbuntuCatalogLoader(factory, cache, fileSystem, NullLogger.Instance, new AdjustableTimeProvider()); + + var options = new UbuntuConnectorOptions + { + IndexUri = new Uri("https://ubuntu.com/security/csaf/index.json"), + OfflineSnapshotPath = "/snapshots/ubuntu-index.json", + }; + + var result = await loader.LoadAsync(options, CancellationToken.None); + result.Metadata.Channels.Should().HaveCount(1); + result.Metadata.Channels[0].Name.Should().Be("stable"); + fileSystem.FileExists(options.OfflineSnapshotPath!).Should().BeTrue(); + result.FromCache.Should().BeFalse(); + + handler.ResetInvocationCount(); + var cached = await loader.LoadAsync(options, CancellationToken.None); + cached.FromCache.Should().BeTrue(); + handler.InvocationCount.Should().Be(0); + } + + [Fact] + public async Task LoadAsync_UsesOfflineSnapshotWhenPreferred() + { + var handler = new TestHttpMessageHandler(new Dictionary()); + var client = new HttpClient(handler); + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + fileSystem.AddFile("/snapshots/ubuntu-index.json", new MockFileData($"{{\"metadata\":{SampleIndex},\"fetchedAt\":\"2025-10-10T00:00:00Z\"}}")); + var loader = new UbuntuCatalogLoader(factory, cache, fileSystem, NullLogger.Instance, new AdjustableTimeProvider()); + + var options = new UbuntuConnectorOptions + { + IndexUri = new Uri("https://ubuntu.com/security/csaf/index.json"), + OfflineSnapshotPath = "/snapshots/ubuntu-index.json", + PreferOfflineSnapshot = true, + Channels = { "stable" } + }; + + var result = await loader.LoadAsync(options, CancellationToken.None); + result.FromOfflineSnapshot.Should().BeTrue(); + result.Metadata.Channels.Should().NotBeEmpty(); + } + + [Fact] + public async Task LoadAsync_ThrowsWhenNoChannelsMatch() + { + var handler = new TestHttpMessageHandler(new Dictionary + { + [new Uri("https://ubuntu.com/security/csaf/index.json")] = CreateResponse(SampleIndex), + }); + var client = new HttpClient(handler); + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var loader = new UbuntuCatalogLoader(factory, cache, fileSystem, NullLogger.Instance, new AdjustableTimeProvider()); + + var options = new UbuntuConnectorOptions + { + IndexUri = new Uri("https://ubuntu.com/security/csaf/index.json"), + }; + options.Channels.Clear(); + options.Channels.Add("nonexistent"); + + await Assert.ThrowsAsync(() => loader.LoadAsync(options, CancellationToken.None)); + } + + private static HttpResponseMessage CreateResponse(string payload) + => new(HttpStatusCode.OK) + { + Content = new StringContent(payload, Encoding.UTF8, "application/json"), + }; + + private sealed class SingleClientHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleClientHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class AdjustableTimeProvider : TimeProvider + { + private DateTimeOffset _now = DateTimeOffset.UtcNow; + + public override DateTimeOffset GetUtcNow() => _now; + } + + private sealed class TestHttpMessageHandler : HttpMessageHandler + { + private readonly Dictionary _responses; + + public TestHttpMessageHandler(Dictionary responses) + { + _responses = responses; + } + + public int InvocationCount { get; private set; } + + public void ResetInvocationCount() => InvocationCount = 0; + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + InvocationCount++; + if (request.RequestUri is not null && _responses.TryGetValue(request.RequestUri, out var response)) + { + var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + return new HttpResponseMessage(response.StatusCode) + { + Content = new StringContent(payload, Encoding.UTF8, "application/json"), + }; + } + + return new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = new StringContent("unexpected request"), + }; + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests/StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests.csproj b/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests/StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests.csproj new file mode 100644 index 00000000..68fa2841 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests/StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests.csproj @@ -0,0 +1,17 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptions.cs b/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptions.cs new file mode 100644 index 00000000..d575b61c --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptions.cs @@ -0,0 +1,90 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Abstractions; + +namespace StellaOps.Vexer.Connectors.Ubuntu.CSAF.Configuration; + +public sealed class UbuntuConnectorOptions +{ + public const string HttpClientName = "vexer.connector.ubuntu.catalog"; + + /// + /// Root index that lists Ubuntu CSAF channels. + /// + public Uri IndexUri { get; set; } = new("https://ubuntu.com/security/csaf/index.json"); + + /// + /// Channels to include (e.g. stable, esm, lts). + /// + public IList Channels { get; } = new List { "stable" }; + + /// + /// Duration to cache discovery metadata. + /// + public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromHours(4); + + /// + /// Prefer offline snapshot when available. + /// + public bool PreferOfflineSnapshot { get; set; } + /// + /// Optional file path for offline index snapshot. + /// + public string? OfflineSnapshotPath { get; set; } + /// + /// Controls persistence of network responses to . + /// + public bool PersistOfflineSnapshot { get; set; } = true; + + public void Validate(IFileSystem? fileSystem = null) + { + if (IndexUri is null || !IndexUri.IsAbsoluteUri) + { + throw new InvalidOperationException("IndexUri must be an absolute URI."); + } + + if (IndexUri.Scheme is not ("http" or "https")) + { + throw new InvalidOperationException("IndexUri must use HTTP or HTTPS."); + } + + if (Channels.Count == 0) + { + throw new InvalidOperationException("At least one channel must be specified."); + } + + for (var i = Channels.Count - 1; i >= 0; i--) + { + if (string.IsNullOrWhiteSpace(Channels[i])) + { + Channels.RemoveAt(i); + } + } + + if (Channels.Count == 0) + { + throw new InvalidOperationException("Channel names cannot be empty."); + } + + if (MetadataCacheDuration <= TimeSpan.Zero) + { + throw new InvalidOperationException("MetadataCacheDuration must be positive."); + } + + if (PreferOfflineSnapshot && string.IsNullOrWhiteSpace(OfflineSnapshotPath)) + { + throw new InvalidOperationException("OfflineSnapshotPath must be provided when PreferOfflineSnapshot is enabled."); + } + + if (!string.IsNullOrWhiteSpace(OfflineSnapshotPath)) + { + var fs = fileSystem ?? new FileSystem(); + var directory = Path.GetDirectoryName(OfflineSnapshotPath); + if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory)) + { + fs.Directory.CreateDirectory(directory); + } + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptionsValidator.cs b/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptionsValidator.cs new file mode 100644 index 00000000..1adb98fd --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptionsValidator.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using System.IO.Abstractions; +using StellaOps.Vexer.Connectors.Abstractions; + +namespace StellaOps.Vexer.Connectors.Ubuntu.CSAF.Configuration; + +public sealed class UbuntuConnectorOptionsValidator : IVexConnectorOptionsValidator +{ + private readonly IFileSystem _fileSystem; + + public UbuntuConnectorOptionsValidator(IFileSystem fileSystem) + { + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + } + + public void Validate(VexConnectorDescriptor descriptor, UbuntuConnectorOptions options, IList errors) + { + ArgumentNullException.ThrowIfNull(descriptor); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(errors); + + try + { + options.Validate(_fileSystem); + } + catch (Exception ex) + { + errors.Add(ex.Message); + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/DependencyInjection/UbuntuConnectorServiceCollectionExtensions.cs b/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/DependencyInjection/UbuntuConnectorServiceCollectionExtensions.cs new file mode 100644 index 00000000..42d759c8 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/DependencyInjection/UbuntuConnectorServiceCollectionExtensions.cs @@ -0,0 +1,45 @@ +using System; +using System.Net; +using System.Net.Http; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using StellaOps.Vexer.Connectors.Abstractions; +using StellaOps.Vexer.Connectors.Ubuntu.CSAF.Configuration; +using StellaOps.Vexer.Connectors.Ubuntu.CSAF.Metadata; +using StellaOps.Vexer.Core; +using System.IO.Abstractions; + +namespace StellaOps.Vexer.Connectors.Ubuntu.CSAF.DependencyInjection; + +public static class UbuntuConnectorServiceCollectionExtensions +{ + public static IServiceCollection AddUbuntuCsafConnector(this IServiceCollection services, Action? configure = null) + { + ArgumentNullException.ThrowIfNull(services); + + services.TryAddSingleton(); + services.TryAddSingleton(); + + services.AddOptions() + .Configure(options => configure?.Invoke(options)); + + services.AddSingleton, UbuntuConnectorOptionsValidator>(); + + services.AddHttpClient(UbuntuConnectorOptions.HttpClientName, client => + { + client.Timeout = TimeSpan.FromSeconds(60); + client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Vexer.Connectors.Ubuntu.CSAF/1.0"); + client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); + }) + .ConfigurePrimaryHttpMessageHandler(static () => new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.All, + }); + + services.AddSingleton(); + services.AddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/Metadata/UbuntuCatalogLoader.cs b/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/Metadata/UbuntuCatalogLoader.cs new file mode 100644 index 00000000..890c3420 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/Metadata/UbuntuCatalogLoader.cs @@ -0,0 +1,248 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO.Abstractions; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using StellaOps.Vexer.Connectors.Ubuntu.CSAF.Configuration; + +namespace StellaOps.Vexer.Connectors.Ubuntu.CSAF.Metadata; + +public sealed class UbuntuCatalogLoader +{ + public const string CachePrefix = "StellaOps.Vexer.Connectors.Ubuntu.CSAF.Index"; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly IMemoryCache _memoryCache; + private readonly IFileSystem _fileSystem; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly SemaphoreSlim _semaphore = new(1, 1); + private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web); + + public UbuntuCatalogLoader( + IHttpClientFactory httpClientFactory, + IMemoryCache memoryCache, + IFileSystem fileSystem, + ILogger logger, + TimeProvider? timeProvider = null) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache)); + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public async Task LoadAsync(UbuntuConnectorOptions options, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + options.Validate(_fileSystem); + + var cacheKey = CreateCacheKey(options); + if (_memoryCache.TryGetValue(cacheKey, out var cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow())) + { + return cached.ToResult(fromCache: true); + } + + await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_memoryCache.TryGetValue(cacheKey, out cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow())) + { + return cached.ToResult(fromCache: true); + } + + CacheEntry? entry = null; + if (options.PreferOfflineSnapshot) + { + entry = LoadFromOffline(options); + if (entry is null) + { + throw new InvalidOperationException("PreferOfflineSnapshot is enabled but no offline Ubuntu snapshot was found."); + } + } + else + { + entry = await TryFetchFromNetworkAsync(options, cancellationToken).ConfigureAwait(false) + ?? LoadFromOffline(options); + } + + if (entry is null) + { + throw new InvalidOperationException("Unable to load Ubuntu CSAF index from network or offline snapshot."); + } + + var cacheOptions = new MemoryCacheEntryOptions(); + if (entry.MetadataCacheDuration > TimeSpan.Zero) + { + cacheOptions.AbsoluteExpiration = _timeProvider.GetUtcNow().Add(entry.MetadataCacheDuration); + } + + _memoryCache.Set(cacheKey, entry with { CachedAt = _timeProvider.GetUtcNow() }, cacheOptions); + return entry.ToResult(fromCache: false); + } + finally + { + _semaphore.Release(); + } + } + + private async Task TryFetchFromNetworkAsync(UbuntuConnectorOptions options, CancellationToken cancellationToken) + { + try + { + var client = _httpClientFactory.CreateClient(UbuntuConnectorOptions.HttpClientName); + using var response = await client.GetAsync(options.IndexUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + + var metadata = ParseMetadata(payload, options.Channels); + var now = _timeProvider.GetUtcNow(); + var entry = new CacheEntry(metadata, now, now, options.MetadataCacheDuration, false); + + PersistSnapshotIfNeeded(options, metadata, now); + return entry; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, "Failed to fetch Ubuntu CSAF index from {Uri}; attempting offline fallback if available.", options.IndexUri); + return null; + } + } + + private CacheEntry? LoadFromOffline(UbuntuConnectorOptions options) + { + if (string.IsNullOrWhiteSpace(options.OfflineSnapshotPath)) + { + return null; + } + + if (!_fileSystem.File.Exists(options.OfflineSnapshotPath)) + { + _logger.LogWarning("Ubuntu offline snapshot path {Path} does not exist.", options.OfflineSnapshotPath); + return null; + } + + try + { + var payload = _fileSystem.File.ReadAllText(options.OfflineSnapshotPath); + var snapshot = JsonSerializer.Deserialize(payload, _serializerOptions); + if (snapshot is null) + { + throw new InvalidOperationException("Offline snapshot payload was empty."); + } + + return new CacheEntry(snapshot.Metadata, snapshot.FetchedAt, _timeProvider.GetUtcNow(), options.MetadataCacheDuration, true); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to load Ubuntu CSAF index from offline snapshot {Path}.", options.OfflineSnapshotPath); + return null; + } + } + + private UbuntuCatalogMetadata ParseMetadata(string payload, IList channels) + { + if (string.IsNullOrWhiteSpace(payload)) + { + throw new InvalidOperationException("Ubuntu index payload was empty."); + } + + using var document = JsonDocument.Parse(payload); + var root = document.RootElement; + + var generatedAt = root.TryGetProperty("generated", out var generatedElement) && generatedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(generatedElement.GetString(), out var generated) + ? generated + : _timeProvider.GetUtcNow(); + + var channelSet = new HashSet(channels, StringComparer.OrdinalIgnoreCase); + + if (!root.TryGetProperty("channels", out var channelsElement) || channelsElement.ValueKind is not JsonValueKind.Array) + { + throw new InvalidOperationException("Ubuntu index did not include a channels array."); + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var channelElement in channelsElement.EnumerateArray()) + { + var name = channelElement.TryGetProperty("name", out var nameElement) && nameElement.ValueKind == JsonValueKind.String ? nameElement.GetString() : null; + if (string.IsNullOrWhiteSpace(name) || !channelSet.Contains(name)) + { + continue; + } + + if (!channelElement.TryGetProperty("catalogUrl", out var urlElement) || urlElement.ValueKind != JsonValueKind.String || !Uri.TryCreate(urlElement.GetString(), UriKind.Absolute, out var catalogUri)) + { + _logger.LogWarning("Channel {Channel} did not specify a valid catalogUrl.", name); + continue; + } + + string? sha256 = null; + if (channelElement.TryGetProperty("sha256", out var shaElement) && shaElement.ValueKind == JsonValueKind.String) + { + sha256 = shaElement.GetString(); + } + + DateTimeOffset? lastUpdated = null; + if (channelElement.TryGetProperty("lastUpdated", out var updatedElement) && updatedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(updatedElement.GetString(), out var updated)) + { + lastUpdated = updated; + } + + builder.Add(new UbuChannelCatalog(name!, catalogUri, sha256, lastUpdated)); + } + + if (builder.Count == 0) + { + throw new InvalidOperationException("None of the requested Ubuntu channels were present in the index."); + } + + return new UbuntuCatalogMetadata(generatedAt, builder.ToImmutable()); + } + + private void PersistSnapshotIfNeeded(UbuntuConnectorOptions options, UbuntuCatalogMetadata metadata, DateTimeOffset fetchedAt) + { + if (!options.PersistOfflineSnapshot || string.IsNullOrWhiteSpace(options.OfflineSnapshotPath)) + { + return; + } + + try + { + var snapshot = new UbuntuCatalogSnapshot(metadata, fetchedAt); + var payload = JsonSerializer.Serialize(snapshot, _serializerOptions); + _fileSystem.File.WriteAllText(options.OfflineSnapshotPath, payload); + _logger.LogDebug("Persisted Ubuntu CSAF index snapshot to {Path}.", options.OfflineSnapshotPath); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to persist Ubuntu CSAF index snapshot to {Path}.", options.OfflineSnapshotPath); + } + } + + private static string CreateCacheKey(UbuntuConnectorOptions options) + => $"{CachePrefix}:{options.IndexUri}:{string.Join(',', options.Channels)}"; + + private sealed record CacheEntry(UbuntuCatalogMetadata Metadata, DateTimeOffset FetchedAt, DateTimeOffset CachedAt, TimeSpan MetadataCacheDuration, bool FromOfflineSnapshot) + { + public bool IsExpired(DateTimeOffset now) + => MetadataCacheDuration > TimeSpan.Zero && now >= CachedAt + MetadataCacheDuration; + + public UbuntuCatalogResult ToResult(bool fromCache) + => new(Metadata, FetchedAt, fromCache, FromOfflineSnapshot); + } + + private sealed record UbuntuCatalogSnapshot(UbuntuCatalogMetadata Metadata, DateTimeOffset FetchedAt); +} + +public sealed record UbuntuCatalogMetadata(DateTimeOffset GeneratedAt, ImmutableArray Channels); + +public sealed record UbuChannelCatalog(string Name, Uri CatalogUri, string? Sha256, DateTimeOffset? LastUpdated); + +public sealed record UbuntuCatalogResult(UbuntuCatalogMetadata Metadata, DateTimeOffset FetchedAt, bool FromCache, bool FromOfflineSnapshot); diff --git a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/StellaOps.Vexer.Connectors.Ubuntu.CSAF.csproj b/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/StellaOps.Vexer.Connectors.Ubuntu.CSAF.csproj new file mode 100644 index 00000000..a99a942f --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/StellaOps.Vexer.Connectors.Ubuntu.CSAF.csproj @@ -0,0 +1,18 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/TASKS.md b/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/TASKS.md index b0e8e146..049b0920 100644 --- a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/TASKS.md +++ b/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/TASKS.md @@ -2,6 +2,6 @@ If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and # TASKS | Task | Owner(s) | Depends on | Notes | |---|---|---|---| -|VEXER-CONN-UBUNTU-01-001 – Ubuntu CSAF discovery & channels|Team Vexer Connectors – Ubuntu|VEXER-CONN-ABS-01-001|TODO – Implement discovery of Ubuntu CSAF catalogs, channel selection (stable/LTS), and offline snapshot import.| +|VEXER-CONN-UBUNTU-01-001 – Ubuntu CSAF discovery & channels|Team Vexer Connectors – Ubuntu|VEXER-CONN-ABS-01-001|**DONE (2025-10-17)** – Added Ubuntu connector project with configurable channel options, catalog loader (network/offline), DI wiring, and discovery unit tests.| |VEXER-CONN-UBUNTU-01-002 – Incremental fetch & deduplication|Team Vexer Connectors – Ubuntu|VEXER-CONN-UBUNTU-01-001, VEXER-STORAGE-01-003|TODO – Fetch CSAF bundles with ETag handling, checksum validation, deduplication, and raw persistence.| |VEXER-CONN-UBUNTU-01-003 – Trust metadata & provenance|Team Vexer Connectors – Ubuntu|VEXER-CONN-UBUNTU-01-002, VEXER-POLICY-01-001|TODO – Emit Ubuntu signing metadata (GPG fingerprints) plus provenance hints for policy weighting and diagnostics.| diff --git a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/UbuntuCsafConnector.cs b/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/UbuntuCsafConnector.cs new file mode 100644 index 00000000..c47702f8 --- /dev/null +++ b/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/UbuntuCsafConnector.cs @@ -0,0 +1,80 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.Logging; +using StellaOps.Vexer.Connectors.Abstractions; +using StellaOps.Vexer.Connectors.Ubuntu.CSAF.Configuration; +using StellaOps.Vexer.Connectors.Ubuntu.CSAF.Metadata; +using StellaOps.Vexer.Core; + +namespace StellaOps.Vexer.Connectors.Ubuntu.CSAF; + +public sealed class UbuntuCsafConnector : VexConnectorBase +{ + private static readonly VexConnectorDescriptor DescriptorInstance = new( + id: "vexer:ubuntu", + kind: VexProviderKind.Distro, + displayName: "Ubuntu CSAF") + { + Tags = ImmutableArray.Create("ubuntu", "csaf", "usn"), + }; + + private readonly UbuntuCatalogLoader _catalogLoader; + private readonly IEnumerable> _validators; + + private UbuntuConnectorOptions? _options; + private UbuntuCatalogResult? _catalog; + + public UbuntuCsafConnector( + UbuntuCatalogLoader catalogLoader, + IEnumerable> validators, + ILogger logger, + TimeProvider timeProvider) + : base(DescriptorInstance, logger, timeProvider) + { + _catalogLoader = catalogLoader ?? throw new ArgumentNullException(nameof(catalogLoader)); + _validators = validators ?? Array.Empty>(); + } + + public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) + { + _options = VexConnectorOptionsBinder.Bind( + Descriptor, + settings, + validators: _validators); + + _catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); + LogConnectorEvent(LogLevel.Information, "validate", "Ubuntu CSAF index loaded.", new Dictionary + { + ["channelCount"] = _catalog.Metadata.Channels.Length, + ["fromOffline"] = _catalog.FromOfflineSnapshot, + }); + } + + public override async IAsyncEnumerable FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + if (_options is null) + { + throw new InvalidOperationException("Connector must be validated before fetch operations."); + } + + if (_catalog is null) + { + _catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); + } + + LogConnectorEvent(LogLevel.Debug, "fetch", "Ubuntu CSAF discovery ready; channel catalogs handled in subsequent task.", new Dictionary + { + ["since"] = context.Since?.ToString("O"), + }); + + yield break; + } + + public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => throw new NotSupportedException("UbuntuCsafConnector relies on CSAF normalizers for document processing."); + + public UbuntuCatalogResult? GetCachedCatalog() => _catalog; +} diff --git a/src/StellaOps.Vexer.Export.Tests/ExportEngineTests.cs b/src/StellaOps.Vexer.Export.Tests/ExportEngineTests.cs index fc917b06..e7ecf44c 100644 --- a/src/StellaOps.Vexer.Export.Tests/ExportEngineTests.cs +++ b/src/StellaOps.Vexer.Export.Tests/ExportEngineTests.cs @@ -88,10 +88,46 @@ public sealed class ExportEngineTests Assert.Equal(1, recorder2.SaveCount); } + [Fact] + public async Task ExportAsync_AttachesAttestationMetadata() + { + var store = new InMemoryExportStore(); + var evaluator = new StaticPolicyEvaluator("baseline/v1"); + var dataSource = new InMemoryExportDataSource(); + var exporter = new DummyExporter(VexExportFormat.Json); + var attestation = new RecordingAttestationClient(); + var engine = new VexExportEngine( + store, + evaluator, + dataSource, + new[] { exporter }, + NullLogger.Instance, + cacheIndex: null, + artifactStores: null, + attestationClient: attestation); + + var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") }); + var requestedAt = DateTimeOffset.UtcNow; + var context = new VexExportRequestContext(query, VexExportFormat.Json, requestedAt); + + var manifest = await engine.ExportAsync(context, CancellationToken.None); + + Assert.NotNull(attestation.LastRequest); + Assert.Equal(manifest.ExportId, attestation.LastRequest!.ExportId); + Assert.NotNull(manifest.Attestation); + Assert.Equal(attestation.Response.Attestation.EnvelopeDigest, manifest.Attestation!.EnvelopeDigest); + Assert.Equal(attestation.Response.Attestation.PredicateType, manifest.Attestation.PredicateType); + + Assert.NotNull(store.LastSavedManifest); + Assert.Equal(manifest.Attestation, store.LastSavedManifest!.Attestation); + } + private sealed class InMemoryExportStore : IVexExportStore { private readonly Dictionary _store = new(StringComparer.Ordinal); + public VexExportManifest? LastSavedManifest { get; private set; } + public ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) { var key = CreateKey(signature.Value, format); @@ -103,6 +139,7 @@ public sealed class ExportEngineTests { var key = CreateKey(manifest.QuerySignature.Value, manifest.Format); _store[key] = manifest; + LastSavedManifest = manifest; return ValueTask.CompletedTask; } @@ -110,6 +147,28 @@ public sealed class ExportEngineTests => FormattableString.Invariant($"{signature}|{format}"); } + private sealed class RecordingAttestationClient : IVexAttestationClient + { + public VexAttestationRequest? LastRequest { get; private set; } + + public VexAttestationResponse Response { get; } = new VexAttestationResponse( + new VexAttestationMetadata( + predicateType: "https://stella-ops.org/attestations/vex-export", + rekor: new VexRekorReference("0.2", "rekor://entry", "123"), + envelopeDigest: "sha256:envelope", + signedAt: DateTimeOffset.UnixEpoch), + ImmutableDictionary.Empty); + + public ValueTask SignAsync(VexAttestationRequest request, CancellationToken cancellationToken) + { + LastRequest = request; + return ValueTask.FromResult(Response); + } + + public ValueTask VerifyAsync(VexAttestationRequest request, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexAttestationVerification(true, ImmutableDictionary.Empty)); + } + private sealed class RecordingCacheIndex : IVexCacheIndex { public Dictionary<(string Signature, VexExportFormat Format), bool> RemoveCalls { get; } = new(); @@ -213,4 +272,5 @@ public sealed class ExportEngineTests return ValueTask.FromResult(new VexExportResult(new VexContentAddress("sha256", "deadbeef"), bytes.Length, ImmutableDictionary.Empty)); } } + } diff --git a/src/StellaOps.Vexer.Export/ExportEngine.cs b/src/StellaOps.Vexer.Export/ExportEngine.cs index 3ea66ee6..a93ee805 100644 --- a/src/StellaOps.Vexer.Export/ExportEngine.cs +++ b/src/StellaOps.Vexer.Export/ExportEngine.cs @@ -39,6 +39,7 @@ public sealed class VexExportEngine : IExportEngine private readonly ILogger _logger; private readonly IVexCacheIndex? _cacheIndex; private readonly IReadOnlyList _artifactStores; + private readonly IVexAttestationClient? _attestationClient; public VexExportEngine( IVexExportStore exportStore, @@ -47,7 +48,8 @@ public sealed class VexExportEngine : IExportEngine IEnumerable exporters, ILogger logger, IVexCacheIndex? cacheIndex = null, - IEnumerable? artifactStores = null) + IEnumerable? artifactStores = null, + IVexAttestationClient? attestationClient = null) { _exportStore = exportStore ?? throw new ArgumentNullException(nameof(exportStore)); _policyEvaluator = policyEvaluator ?? throw new ArgumentNullException(nameof(policyEvaluator)); @@ -55,6 +57,7 @@ public sealed class VexExportEngine : IExportEngine _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _cacheIndex = cacheIndex; _artifactStores = artifactStores?.ToArray() ?? Array.Empty(); + _attestationClient = attestationClient; if (exporters is null) { @@ -105,6 +108,7 @@ public sealed class VexExportEngine : IExportEngine context.RequestedAt); var digest = exporter.Digest(exportRequest); + var exportId = FormattableString.Invariant($"exports/{context.RequestedAt:yyyyMMddTHHmmssfffZ}/{digest.Digest}"); await using var buffer = new MemoryStream(); var result = await exporter.SerializeAsync(exportRequest, buffer, cancellationToken).ConfigureAwait(false); @@ -134,7 +138,36 @@ public sealed class VexExportEngine : IExportEngine } } - var exportId = FormattableString.Invariant($"exports/{context.RequestedAt:yyyyMMddTHHmmssfffZ}/{digest.Digest}"); + VexAttestationMetadata? attestationMetadata = null; + if (_attestationClient is not null) + { + var attestationRequest = new VexAttestationRequest( + exportId, + signature, + digest, + context.Format, + context.RequestedAt, + dataset.SourceProviders, + result.Metadata); + + var response = await _attestationClient.SignAsync(attestationRequest, cancellationToken).ConfigureAwait(false); + attestationMetadata = response.Attestation; + + if (!response.Diagnostics.IsEmpty) + { + foreach (var diagnostic in response.Diagnostics) + { + _logger.LogDebug( + "Attestation diagnostic {Key}={Value} for export {ExportId}", + diagnostic.Key, + diagnostic.Value, + exportId); + } + } + + _logger.LogInformation("Attestation generated for export {ExportId}", exportId); + } + var manifest = new VexExportManifest( exportId, signature, @@ -145,7 +178,7 @@ public sealed class VexExportEngine : IExportEngine dataset.SourceProviders, fromCache: false, consensusRevision: _policyEvaluator.Version, - attestation: null, + attestation: attestationMetadata, sizeBytes: result.BytesWritten); await _exportStore.SaveAsync(manifest, cancellationToken).ConfigureAwait(false); diff --git a/src/StellaOps.Vexer.Export/TASKS.md b/src/StellaOps.Vexer.Export/TASKS.md index f229d90d..0e737d33 100644 --- a/src/StellaOps.Vexer.Export/TASKS.md +++ b/src/StellaOps.Vexer.Export/TASKS.md @@ -5,5 +5,5 @@ If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and |VEXER-EXPORT-01-001 – Export engine orchestration|Team Vexer Export|VEXER-CORE-01-003|DONE (2025-10-15) – Export engine scaffolding with cache lookup, data source hooks, and deterministic manifest emission.| |VEXER-EXPORT-01-002 – Cache index & eviction hooks|Team Vexer Export|VEXER-EXPORT-01-001, VEXER-STORAGE-01-003|**DONE (2025-10-16)** – Export engine now invalidates cache entries on force refresh, cache services expose prune/invalidate APIs, and storage maintenance trims expired/dangling records with Mongo2Go coverage.| |VEXER-EXPORT-01-003 – Artifact store adapters|Team Vexer Export|VEXER-EXPORT-01-001|**DONE (2025-10-16)** – Implemented multi-store pipeline with filesystem, S3-compatible, and offline bundle adapters (hash verification + manifest/zip output) plus unit coverage and DI hooks.| -|VEXER-EXPORT-01-004 – Attestation handoff integration|Team Vexer Export|VEXER-EXPORT-01-001, VEXER-ATTEST-01-001|TODO – Connect export engine to attestation client, persist Rekor metadata, and reuse cached attestations.| +|VEXER-EXPORT-01-004 – Attestation handoff integration|Team Vexer Export|VEXER-EXPORT-01-001, VEXER-ATTEST-01-001|**DONE (2025-10-17)** – Export engine now invokes attestation client, logs diagnostics, and persists Rekor/envelope metadata on manifests; regression coverage added in `ExportEngineTests.ExportAsync_AttachesAttestationMetadata`.| |VEXER-EXPORT-01-005 – Score & resolve envelope surfaces|Team Vexer Export|VEXER-EXPORT-01-004, VEXER-CORE-02-001|TODO – Emit consensus+score envelopes in export manifests, include policy/scoring digests, and update offline bundle/ORAS layouts to carry signed VEX responses.| diff --git a/src/StellaOps.Vexer.Formats.CSAF.Tests/CsafNormalizerTests.cs b/src/StellaOps.Vexer.Formats.CSAF.Tests/CsafNormalizerTests.cs new file mode 100644 index 00000000..f14a9d61 --- /dev/null +++ b/src/StellaOps.Vexer.Formats.CSAF.Tests/CsafNormalizerTests.cs @@ -0,0 +1,131 @@ +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.IO; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Vexer.Core; +using StellaOps.Vexer.Formats.CSAF; + +namespace StellaOps.Vexer.Formats.CSAF.Tests; + +public sealed class CsafNormalizerTests +{ + [Fact] + public async Task NormalizeAsync_ProducesClaimsPerProductStatus() + { + var json = """ + { + "document": { + "tracking": { + "id": "RHSA-2025:0001", + "version": "3", + "revision": "3", + "status": "final", + "initial_release_date": "2025-10-01T00:00:00Z", + "current_release_date": "2025-10-10T00:00:00Z" + }, + "publisher": { + "name": "Red Hat Product Security", + "category": "vendor" + } + }, + "product_tree": { + "full_product_names": [ + { + "product_id": "CSAFPID-0001", + "name": "Red Hat Enterprise Linux 9", + "product_identification_helper": { + "cpe": "cpe:/o:redhat:enterprise_linux:9", + "purl": "pkg:rpm/redhat/enterprise-linux@9" + } + } + ] + }, + "vulnerabilities": [ + { + "cve": "CVE-2025-0001", + "title": "Kernel vulnerability", + "product_status": { + "known_affected": [ "CSAFPID-0001" ] + } + }, + { + "id": "VEX-0002", + "title": "Library issue", + "product_status": { + "known_not_affected": [ "CSAFPID-0001" ] + } + } + ] + } + """; + + var rawDocument = new VexRawDocument( + ProviderId: "vexer:redhat", + VexDocumentFormat.Csaf, + new Uri("https://example.com/csaf/rhsa-2025-0001.json"), + new DateTimeOffset(2025, 10, 11, 0, 0, 0, TimeSpan.Zero), + "sha256:dummydigest", + Encoding.UTF8.GetBytes(json), + ImmutableDictionary.Empty); + + var provider = new VexProvider("vexer:redhat", "Red Hat CSAF", VexProviderKind.Distro); + var normalizer = new CsafNormalizer(NullLogger.Instance); + + var batch = await normalizer.NormalizeAsync(rawDocument, provider, CancellationToken.None); + + batch.Claims.Should().HaveCount(2); + + var affectedClaim = batch.Claims.First(c => c.VulnerabilityId == "CVE-2025-0001"); + affectedClaim.Status.Should().Be(VexClaimStatus.Affected); + affectedClaim.Product.Key.Should().Be("CSAFPID-0001"); + affectedClaim.Product.Purl.Should().Be("pkg:rpm/redhat/enterprise-linux@9"); + affectedClaim.Product.Cpe.Should().Be("cpe:/o:redhat:enterprise_linux:9"); + affectedClaim.Document.Revision.Should().Be("3"); + affectedClaim.FirstSeen.Should().Be(new DateTimeOffset(2025, 10, 1, 0, 0, 0, TimeSpan.Zero)); + affectedClaim.LastSeen.Should().Be(new DateTimeOffset(2025, 10, 10, 0, 0, 0, TimeSpan.Zero)); + affectedClaim.AdditionalMetadata.Should().ContainKey("csaf.tracking.id"); + affectedClaim.AdditionalMetadata["csaf.publisher.name"].Should().Be("Red Hat Product Security"); + + var notAffectedClaim = batch.Claims.First(c => c.VulnerabilityId == "VEX-0002"); + notAffectedClaim.Status.Should().Be(VexClaimStatus.NotAffected); + } + + [Fact] + public async Task NormalizeAsync_PreservesRedHatSpecificMetadata() + { + var path = Path.Combine(AppContext.BaseDirectory, "Fixtures", "rhsa-sample.json"); + var json = await File.ReadAllTextAsync(path); + + var rawDocument = new VexRawDocument( + ProviderId: "vexer:redhat", + VexDocumentFormat.Csaf, + new Uri("https://security.example.com/rhsa-2025-1001.json"), + new DateTimeOffset(2025, 10, 6, 0, 0, 0, TimeSpan.Zero), + "sha256:rhdadigest", + Encoding.UTF8.GetBytes(json), + ImmutableDictionary.Empty); + + var provider = new VexProvider("vexer:redhat", "Red Hat CSAF", VexProviderKind.Distro); + var normalizer = new CsafNormalizer(NullLogger.Instance); + + var batch = await normalizer.NormalizeAsync(rawDocument, provider, CancellationToken.None); + batch.Claims.Should().ContainSingle(); + + var claim = batch.Claims[0]; + claim.VulnerabilityId.Should().Be("CVE-2025-1234"); + claim.Status.Should().Be(VexClaimStatus.Affected); + claim.Product.Key.Should().Be("rh-enterprise-linux-9"); + claim.Product.Name.Should().Be("Red Hat Enterprise Linux 9"); + claim.Product.Purl.Should().Be("pkg:rpm/redhat/enterprise-linux@9"); + claim.Product.Cpe.Should().Be("cpe:/o:redhat:enterprise_linux:9"); + claim.FirstSeen.Should().Be(new DateTimeOffset(2025, 10, 1, 12, 0, 0, TimeSpan.Zero)); + claim.LastSeen.Should().Be(new DateTimeOffset(2025, 10, 5, 10, 0, 0, TimeSpan.Zero)); + + claim.AdditionalMetadata.Should().ContainKey("csaf.tracking.id"); + claim.AdditionalMetadata["csaf.tracking.id"].Should().Be("RHSA-2025:1001"); + claim.AdditionalMetadata["csaf.tracking.status"].Should().Be("final"); + claim.AdditionalMetadata["csaf.publisher.name"].Should().Be("Red Hat Product Security"); + } +} diff --git a/src/StellaOps.Vexer.Formats.CSAF.Tests/Fixtures/rhsa-sample.json b/src/StellaOps.Vexer.Formats.CSAF.Tests/Fixtures/rhsa-sample.json new file mode 100644 index 00000000..65be9b12 --- /dev/null +++ b/src/StellaOps.Vexer.Formats.CSAF.Tests/Fixtures/rhsa-sample.json @@ -0,0 +1,47 @@ +{ + "document": { + "publisher": { + "name": "Red Hat Product Security", + "category": "vendor" + }, + "tracking": { + "id": "RHSA-2025:1001", + "status": "final", + "version": "3", + "initial_release_date": "2025-10-01T12:00:00Z", + "current_release_date": "2025-10-05T10:00:00Z" + } + }, + "product_tree": { + "full_product_names": [ + { + "product_id": "rh-enterprise-linux-9", + "name": "Red Hat Enterprise Linux 9", + "product_identification_helper": { + "cpe": "cpe:/o:redhat:enterprise_linux:9", + "purl": "pkg:rpm/redhat/enterprise-linux@9" + } + } + ], + "branches": [ + { + "name": "Red Hat Enterprise Linux", + "product": { + "product_id": "rh-enterprise-linux-9", + "name": "Red Hat Enterprise Linux 9" + } + } + ] + }, + "vulnerabilities": [ + { + "cve": "CVE-2025-1234", + "title": "Kernel privilege escalation", + "product_status": { + "known_affected": [ + "rh-enterprise-linux-9" + ] + } + } + ] +} diff --git a/src/StellaOps.Vexer.Formats.CSAF.Tests/StellaOps.Vexer.Formats.CSAF.Tests.csproj b/src/StellaOps.Vexer.Formats.CSAF.Tests/StellaOps.Vexer.Formats.CSAF.Tests.csproj new file mode 100644 index 00000000..013f6a75 --- /dev/null +++ b/src/StellaOps.Vexer.Formats.CSAF.Tests/StellaOps.Vexer.Formats.CSAF.Tests.csproj @@ -0,0 +1,20 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Formats.CSAF/CsafNormalizer.cs b/src/StellaOps.Vexer.Formats.CSAF/CsafNormalizer.cs new file mode 100644 index 00000000..342ed295 --- /dev/null +++ b/src/StellaOps.Vexer.Formats.CSAF/CsafNormalizer.cs @@ -0,0 +1,532 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Vexer.Core; + +namespace StellaOps.Vexer.Formats.CSAF; + +public sealed class CsafNormalizer : IVexNormalizer +{ + private static readonly ImmutableDictionary StatusPrecedence = new Dictionary + { + [VexClaimStatus.UnderInvestigation] = 0, + [VexClaimStatus.Affected] = 1, + [VexClaimStatus.NotAffected] = 2, + [VexClaimStatus.Fixed] = 3, + }.ToImmutableDictionary(); + + private readonly ILogger _logger; + + public CsafNormalizer(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string Format => VexDocumentFormat.Csaf.ToString().ToLowerInvariant(); + + public bool CanHandle(VexRawDocument document) + => document is not null && document.Format == VexDocumentFormat.Csaf; + + public ValueTask NormalizeAsync(VexRawDocument document, VexProvider provider, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(provider); + + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var result = CsafParser.Parse(document); + var claims = ImmutableArray.CreateBuilder(result.Claims.Length); + foreach (var entry in result.Claims) + { + var product = new VexProduct( + entry.Product.ProductId, + entry.Product.Name, + entry.Product.Version, + entry.Product.Purl, + entry.Product.Cpe); + + var claimDocument = new VexClaimDocument( + VexDocumentFormat.Csaf, + document.Digest, + document.SourceUri, + result.Revision, + signature: null); + + var claim = new VexClaim( + entry.VulnerabilityId, + provider.Id, + product, + entry.Status, + claimDocument, + result.FirstRelease, + result.LastRelease, + justification: null, + detail: entry.Detail, + confidence: null, + additionalMetadata: result.Metadata); + + claims.Add(claim); + } + + var orderedClaims = claims + .ToImmutable() + .OrderBy(static claim => claim.VulnerabilityId, StringComparer.Ordinal) + .ThenBy(static claim => claim.Product.Key, StringComparer.Ordinal) + .ToImmutableArray(); + + _logger.LogInformation( + "Normalized CSAF document {Source} into {ClaimCount} claim(s).", + document.SourceUri, + orderedClaims.Length); + + return ValueTask.FromResult(new VexClaimBatch( + document, + orderedClaims, + ImmutableDictionary.Empty)); + } + catch (JsonException ex) + { + _logger.LogError(ex, "Failed to parse CSAF document {SourceUri}", document.SourceUri); + throw; + } + } + + private static class CsafParser + { + public static CsafParseResult Parse(VexRawDocument document) + { + using var json = JsonDocument.Parse(document.Content.ToArray()); + var root = json.RootElement; + + var tracking = root.TryGetProperty("document", out var documentElement) && + documentElement.ValueKind == JsonValueKind.Object && + documentElement.TryGetProperty("tracking", out var trackingElement) + ? trackingElement + : default; + + var firstRelease = ParseDate(tracking, "initial_release_date") ?? document.RetrievedAt; + var lastRelease = ParseDate(tracking, "current_release_date") ?? firstRelease; + + if (lastRelease < firstRelease) + { + lastRelease = firstRelease; + } + + var metadataBuilder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + AddIfPresent(metadataBuilder, tracking, "id", "csaf.tracking.id"); + AddIfPresent(metadataBuilder, tracking, "version", "csaf.tracking.version"); + AddIfPresent(metadataBuilder, tracking, "status", "csaf.tracking.status"); + AddPublisherMetadata(metadataBuilder, documentElement); + + var revision = TryGetString(tracking, "revision"); + + var productCatalog = CollectProducts(root); + var claimsBuilder = ImmutableArray.CreateBuilder(); + + if (root.TryGetProperty("vulnerabilities", out var vulnerabilitiesElement) && + vulnerabilitiesElement.ValueKind == JsonValueKind.Array) + { + foreach (var vulnerability in vulnerabilitiesElement.EnumerateArray()) + { + var vulnerabilityId = ResolveVulnerabilityId(vulnerability); + if (string.IsNullOrWhiteSpace(vulnerabilityId)) + { + continue; + } + + var detail = ResolveDetail(vulnerability); + var productClaims = BuildClaimsForVulnerability( + vulnerabilityId, + vulnerability, + productCatalog, + detail); + + claimsBuilder.AddRange(productClaims); + } + } + + return new CsafParseResult( + firstRelease, + lastRelease, + revision, + metadataBuilder.ToImmutable(), + claimsBuilder.ToImmutable()); + } + + private static IReadOnlyList BuildClaimsForVulnerability( + string vulnerabilityId, + JsonElement vulnerability, + IReadOnlyDictionary productCatalog, + string? detail) + { + if (!vulnerability.TryGetProperty("product_status", out var statusElement) || + statusElement.ValueKind != JsonValueKind.Object) + { + return Array.Empty(); + } + + var claims = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var statusProperty in statusElement.EnumerateObject()) + { + var status = MapStatus(statusProperty.Name); + if (status is null) + { + continue; + } + + if (statusProperty.Value.ValueKind != JsonValueKind.Array) + { + continue; + } + + foreach (var productIdElement in statusProperty.Value.EnumerateArray()) + { + var productId = productIdElement.GetString(); + if (string.IsNullOrWhiteSpace(productId)) + { + continue; + } + + var product = ResolveProduct(productCatalog, productId); + UpdateClaim(claims, product, status.Value, detail); + } + } + + if (claims.Count == 0) + { + return Array.Empty(); + } + + return claims.Values + .Select(builder => new CsafClaimEntry( + vulnerabilityId, + builder.Product, + builder.Status, + builder.Detail)) + .ToArray(); + } + + private static void UpdateClaim( + IDictionary claims, + CsafProductInfo product, + VexClaimStatus status, + string? detail) + { + if (!claims.TryGetValue(product.ProductId, out var existing) || + StatusPrecedence[status] > StatusPrecedence[existing.Status]) + { + claims[product.ProductId] = new CsafClaimEntryBuilder(product, status, detail); + } + } + + private static CsafProductInfo ResolveProduct( + IReadOnlyDictionary catalog, + string productId) + { + if (catalog.TryGetValue(productId, out var product)) + { + return product; + } + + return new CsafProductInfo(productId, productId, null, null, null); + } + + private static string ResolveVulnerabilityId(JsonElement vulnerability) + { + var id = TryGetString(vulnerability, "cve") + ?? TryGetString(vulnerability, "id") + ?? TryGetString(vulnerability, "vuln_id"); + + return string.IsNullOrWhiteSpace(id) ? string.Empty : id.Trim(); + } + + private static string? ResolveDetail(JsonElement vulnerability) + { + var title = TryGetString(vulnerability, "title"); + if (!string.IsNullOrWhiteSpace(title)) + { + return title.Trim(); + } + + if (vulnerability.TryGetProperty("notes", out var notesElement) && + notesElement.ValueKind == JsonValueKind.Array) + { + foreach (var note in notesElement.EnumerateArray()) + { + if (note.ValueKind != JsonValueKind.Object) + { + continue; + } + + var category = TryGetString(note, "category"); + if (!string.IsNullOrWhiteSpace(category) && + !string.Equals(category, "description", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var text = TryGetString(note, "text"); + if (!string.IsNullOrWhiteSpace(text)) + { + return text.Trim(); + } + } + } + + return null; + } + + private static Dictionary CollectProducts(JsonElement root) + { + var products = new Dictionary(StringComparer.OrdinalIgnoreCase); + + if (!root.TryGetProperty("product_tree", out var productTree) || + productTree.ValueKind != JsonValueKind.Object) + { + return products; + } + + if (productTree.TryGetProperty("full_product_names", out var fullNames) && + fullNames.ValueKind == JsonValueKind.Array) + { + foreach (var productEntry in fullNames.EnumerateArray()) + { + var product = ParseProduct(productEntry, parentBranchName: null); + if (product is not null) + { + AddOrUpdate(product); + } + } + } + + if (productTree.TryGetProperty("branches", out var branches) && + branches.ValueKind == JsonValueKind.Array) + { + foreach (var branch in branches.EnumerateArray()) + { + VisitBranch(branch, parentBranchName: null); + } + } + + return products; + + void VisitBranch(JsonElement branch, string? parentBranchName) + { + if (branch.ValueKind != JsonValueKind.Object) + { + return; + } + + var branchName = TryGetString(branch, "name") ?? parentBranchName; + + if (branch.TryGetProperty("product", out var productElement)) + { + var product = ParseProduct(productElement, branchName); + if (product is not null) + { + AddOrUpdate(product); + } + } + + if (branch.TryGetProperty("branches", out var childBranches) && + childBranches.ValueKind == JsonValueKind.Array) + { + foreach (var childBranch in childBranches.EnumerateArray()) + { + VisitBranch(childBranch, branchName); + } + } + } + + void AddOrUpdate(CsafProductInfo product) + { + if (products.TryGetValue(product.ProductId, out var existing)) + { + products[product.ProductId] = MergeProducts(existing, product); + } + else + { + products[product.ProductId] = product; + } + } + + static CsafProductInfo MergeProducts(CsafProductInfo existing, CsafProductInfo incoming) + { + static string ChooseName(string incoming, string fallback) + => string.IsNullOrWhiteSpace(incoming) ? fallback : incoming; + + static string? ChooseOptional(string? incoming, string? fallback) + => string.IsNullOrWhiteSpace(incoming) ? fallback : incoming; + + return new CsafProductInfo( + existing.ProductId, + ChooseName(incoming.Name, existing.Name), + ChooseOptional(incoming.Version, existing.Version), + ChooseOptional(incoming.Purl, existing.Purl), + ChooseOptional(incoming.Cpe, existing.Cpe)); + } + } + + private static CsafProductInfo? ParseProduct(JsonElement element, string? parentBranchName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + JsonElement productElement = element; + if (!element.TryGetProperty("product_id", out var idElement) && + element.TryGetProperty("product", out var nestedProduct) && + nestedProduct.ValueKind == JsonValueKind.Object && + nestedProduct.TryGetProperty("product_id", out idElement)) + { + productElement = nestedProduct; + } + + var productId = idElement.GetString(); + if (string.IsNullOrWhiteSpace(productId)) + { + return null; + } + + var name = TryGetString(productElement, "name") + ?? TryGetString(element, "name") + ?? parentBranchName + ?? productId; + + var version = TryGetString(productElement, "product_version") + ?? TryGetString(productElement, "version") + ?? TryGetString(element, "product_version"); + + string? cpe = null; + string? purl = null; + if (productElement.TryGetProperty("product_identification_helper", out var helper) && + helper.ValueKind == JsonValueKind.Object) + { + cpe = TryGetString(helper, "cpe"); + purl = TryGetString(helper, "purl"); + } + + return new CsafProductInfo(productId.Trim(), name.Trim(), version?.Trim(), purl?.Trim(), cpe?.Trim()); + } + + private static VexClaimStatus? MapStatus(string statusName) + { + if (string.IsNullOrWhiteSpace(statusName)) + { + return null; + } + + return statusName switch + { + "known_affected" or "fixed_after_release" or "first_affected" or "last_affected" => VexClaimStatus.Affected, + "known_not_affected" or "last_not_affected" or "first_not_affected" => VexClaimStatus.NotAffected, + "fixed" or "first_fixed" or "last_fixed" => VexClaimStatus.Fixed, + "under_investigation" or "investigating" => VexClaimStatus.UnderInvestigation, + _ => null, + }; + } + + private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (!element.TryGetProperty(propertyName, out var dateElement)) + { + return null; + } + + var value = dateElement.GetString(); + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + if (DateTimeOffset.TryParse(value, out var parsed)) + { + return parsed; + } + + return null; + } + + private static string? TryGetString(JsonElement element, string propertyName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (!element.TryGetProperty(propertyName, out var property)) + { + return null; + } + + return property.ValueKind == JsonValueKind.String ? property.GetString() : null; + } + + private static void AddIfPresent( + ImmutableDictionary.Builder builder, + JsonElement element, + string propertyName, + string metadataKey) + { + var value = TryGetString(element, propertyName); + if (!string.IsNullOrWhiteSpace(value)) + { + builder[metadataKey] = value.Trim(); + } + } + + private static void AddPublisherMetadata( + ImmutableDictionary.Builder builder, + JsonElement documentElement) + { + if (documentElement.ValueKind != JsonValueKind.Object || + !documentElement.TryGetProperty("publisher", out var publisher) || + publisher.ValueKind != JsonValueKind.Object) + { + return; + } + + AddIfPresent(builder, publisher, "name", "csaf.publisher.name"); + AddIfPresent(builder, publisher, "category", "csaf.publisher.category"); + } + + private readonly record struct CsafClaimEntryBuilder( + CsafProductInfo Product, + VexClaimStatus Status, + string? Detail); + } + + private sealed record CsafParseResult( + DateTimeOffset FirstRelease, + DateTimeOffset LastRelease, + string? Revision, + ImmutableDictionary Metadata, + ImmutableArray Claims); + + private sealed record CsafClaimEntry( + string VulnerabilityId, + CsafProductInfo Product, + VexClaimStatus Status, + string? Detail); + + private sealed record CsafProductInfo( + string ProductId, + string Name, + string? Version, + string? Purl, + string? Cpe); +} diff --git a/src/StellaOps.Vexer.Formats.CSAF/ServiceCollectionExtensions.cs b/src/StellaOps.Vexer.Formats.CSAF/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..8e282c08 --- /dev/null +++ b/src/StellaOps.Vexer.Formats.CSAF/ServiceCollectionExtensions.cs @@ -0,0 +1,14 @@ +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Vexer.Core; + +namespace StellaOps.Vexer.Formats.CSAF; + +public static class CsafFormatsServiceCollectionExtensions +{ + public static IServiceCollection AddCsafNormalizer(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + services.AddSingleton(); + return services; + } +} diff --git a/src/StellaOps.Vexer.Formats.CSAF/StellaOps.Vexer.Formats.CSAF.csproj b/src/StellaOps.Vexer.Formats.CSAF/StellaOps.Vexer.Formats.CSAF.csproj new file mode 100644 index 00000000..54abf95e --- /dev/null +++ b/src/StellaOps.Vexer.Formats.CSAF/StellaOps.Vexer.Formats.CSAF.csproj @@ -0,0 +1,16 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + diff --git a/src/StellaOps.Vexer.Formats.CSAF/TASKS.md b/src/StellaOps.Vexer.Formats.CSAF/TASKS.md index 1b31fb03..7b9f41dd 100644 --- a/src/StellaOps.Vexer.Formats.CSAF/TASKS.md +++ b/src/StellaOps.Vexer.Formats.CSAF/TASKS.md @@ -2,6 +2,6 @@ If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and # TASKS | Task | Owner(s) | Depends on | Notes | |---|---|---|---| -|VEXER-FMT-CSAF-01-001 – CSAF normalizer foundation|Team Vexer Formats|VEXER-CORE-01-001|TODO – Implement CSAF parser covering provider metadata, document tracking, and vulnerability/product mapping into `VexClaim`.| +|VEXER-FMT-CSAF-01-001 – CSAF normalizer foundation|Team Vexer Formats|VEXER-CORE-01-001|**DONE (2025-10-17)** – Implemented CSAF normalizer + DI hook, parsing tracking metadata, product tree branches/full names, and mapping product statuses into canonical `VexClaim`s with baseline precedence. Regression added in `CsafNormalizerTests`.| |VEXER-FMT-CSAF-01-002 – Status/justification mapping|Team Vexer Formats|VEXER-FMT-CSAF-01-001, VEXER-POLICY-01-001|TODO – Normalize CSAF `product_status` + `justification` values into policy-aware enums with audit diagnostics for unsupported codes.| |VEXER-FMT-CSAF-01-003 – CSAF export adapter|Team Vexer Formats|VEXER-EXPORT-01-001, VEXER-FMT-CSAF-01-001|TODO – Provide CSAF export writer producing deterministic documents (per vuln/product) and manifest metadata for attestation.| diff --git a/src/StellaOps.Vexer.Formats.CycloneDX.Tests/CycloneDxNormalizerTests.cs b/src/StellaOps.Vexer.Formats.CycloneDX.Tests/CycloneDxNormalizerTests.cs new file mode 100644 index 00000000..5145b830 --- /dev/null +++ b/src/StellaOps.Vexer.Formats.CycloneDX.Tests/CycloneDxNormalizerTests.cs @@ -0,0 +1,93 @@ +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Vexer.Core; +using StellaOps.Vexer.Formats.CycloneDX; + +namespace StellaOps.Vexer.Formats.CycloneDX.Tests; + +public sealed class CycloneDxNormalizerTests +{ + [Fact] + public async Task NormalizeAsync_MapsAnalysisStateAndJustification() + { + var json = """ + { + "bomFormat": "CycloneDX", + "specVersion": "1.4", + "serialNumber": "urn:uuid:1234", + "version": "7", + "metadata": { + "timestamp": "2025-10-15T12:00:00Z" + }, + "components": [ + { + "bom-ref": "pkg:npm/acme/lib@1.0.0", + "name": "acme-lib", + "version": "1.0.0", + "purl": "pkg:npm/acme/lib@1.0.0" + } + ], + "vulnerabilities": [ + { + "id": "CVE-2025-1000", + "detail": "Library issue", + "analysis": { + "state": "not_affected", + "justification": "code_not_present", + "response": [ "can_not_fix", "will_not_fix" ] + }, + "affects": [ + { "ref": "pkg:npm/acme/lib@1.0.0" } + ] + }, + { + "id": "CVE-2025-1001", + "description": "Investigating impact", + "analysis": { + "state": "in_triage" + }, + "affects": [ + { "ref": "pkg:npm/missing/component@2.0.0" } + ] + } + ] + } + """; + + var rawDocument = new VexRawDocument( + "vexer:cyclonedx", + VexDocumentFormat.CycloneDx, + new Uri("https://example.org/vex.json"), + new DateTimeOffset(2025, 10, 16, 0, 0, 0, TimeSpan.Zero), + "sha256:dummydigest", + Encoding.UTF8.GetBytes(json), + ImmutableDictionary.Empty); + + var provider = new VexProvider("vexer:cyclonedx", "CycloneDX Provider", VexProviderKind.Vendor); + var normalizer = new CycloneDxNormalizer(NullLogger.Instance); + + var batch = await normalizer.NormalizeAsync(rawDocument, provider, CancellationToken.None); + + batch.Claims.Should().HaveCount(2); + + var notAffected = batch.Claims.Single(c => c.VulnerabilityId == "CVE-2025-1000"); + notAffected.Status.Should().Be(VexClaimStatus.NotAffected); + notAffected.Justification.Should().Be(VexJustification.CodeNotPresent); + notAffected.Product.Key.Should().Be("pkg:npm/acme/lib@1.0.0"); + notAffected.Product.Purl.Should().Be("pkg:npm/acme/lib@1.0.0"); + notAffected.Document.Revision.Should().Be("7"); + notAffected.AdditionalMetadata["cyclonedx.specVersion"].Should().Be("1.4"); + notAffected.AdditionalMetadata["cyclonedx.analysis.state"].Should().Be("not_affected"); + notAffected.AdditionalMetadata["cyclonedx.analysis.response"].Should().Be("can_not_fix,will_not_fix"); + + var investigating = batch.Claims.Single(c => c.VulnerabilityId == "CVE-2025-1001"); + investigating.Status.Should().Be(VexClaimStatus.UnderInvestigation); + investigating.Justification.Should().BeNull(); + investigating.Product.Key.Should().Be("pkg:npm/missing/component@2.0.0"); + investigating.Product.Name.Should().Be("pkg:npm/missing/component@2.0.0"); + investigating.AdditionalMetadata.Should().ContainKey("cyclonedx.specVersion"); + } +} diff --git a/src/StellaOps.Vexer.Formats.CycloneDX.Tests/StellaOps.Vexer.Formats.CycloneDX.Tests.csproj b/src/StellaOps.Vexer.Formats.CycloneDX.Tests/StellaOps.Vexer.Formats.CycloneDX.Tests.csproj new file mode 100644 index 00000000..4a6b0f69 --- /dev/null +++ b/src/StellaOps.Vexer.Formats.CycloneDX.Tests/StellaOps.Vexer.Formats.CycloneDX.Tests.csproj @@ -0,0 +1,17 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Formats.CycloneDX/CycloneDxNormalizer.cs b/src/StellaOps.Vexer.Formats.CycloneDX/CycloneDxNormalizer.cs new file mode 100644 index 00000000..c12c0d94 --- /dev/null +++ b/src/StellaOps.Vexer.Formats.CycloneDX/CycloneDxNormalizer.cs @@ -0,0 +1,459 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Vexer.Core; + +namespace StellaOps.Vexer.Formats.CycloneDX; + +public sealed class CycloneDxNormalizer : IVexNormalizer +{ + private static readonly ImmutableDictionary StateMap = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["not_affected"] = VexClaimStatus.NotAffected, + ["resolved"] = VexClaimStatus.Fixed, + ["resolved_with_patches"] = VexClaimStatus.Fixed, + ["resolved_no_fix"] = VexClaimStatus.Fixed, + ["fixed"] = VexClaimStatus.Fixed, + ["affected"] = VexClaimStatus.Affected, + ["known_affected"] = VexClaimStatus.Affected, + ["exploitable"] = VexClaimStatus.Affected, + ["in_triage"] = VexClaimStatus.UnderInvestigation, + ["under_investigation"] = VexClaimStatus.UnderInvestigation, + ["unknown"] = VexClaimStatus.UnderInvestigation, + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + private static readonly ImmutableDictionary JustificationMap = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["code_not_present"] = VexJustification.CodeNotPresent, + ["code_not_reachable"] = VexJustification.CodeNotReachable, + ["component_not_present"] = VexJustification.ComponentNotPresent, + ["component_not_configured"] = VexJustification.ComponentNotConfigured, + ["vulnerable_code_not_present"] = VexJustification.VulnerableCodeNotPresent, + ["vulnerable_code_not_in_execute_path"] = VexJustification.VulnerableCodeNotInExecutePath, + ["vulnerable_code_cannot_be_controlled_by_adversary"] = VexJustification.VulnerableCodeCannotBeControlledByAdversary, + ["inline_mitigations_already_exist"] = VexJustification.InlineMitigationsAlreadyExist, + ["protected_by_mitigating_control"] = VexJustification.ProtectedByMitigatingControl, + ["protected_by_compensating_control"] = VexJustification.ProtectedByCompensatingControl, + ["requires_configuration"] = VexJustification.RequiresConfiguration, + ["requires_dependency"] = VexJustification.RequiresDependency, + ["requires_environment"] = VexJustification.RequiresEnvironment, + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + private readonly ILogger _logger; + + public CycloneDxNormalizer(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string Format => VexDocumentFormat.CycloneDx.ToString().ToLowerInvariant(); + + public bool CanHandle(VexRawDocument document) + => document is not null && document.Format == VexDocumentFormat.CycloneDx; + + public ValueTask NormalizeAsync(VexRawDocument document, VexProvider provider, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(provider); + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var parseResult = CycloneDxParser.Parse(document); + var baseMetadata = parseResult.Metadata; + var claimsBuilder = ImmutableArray.CreateBuilder(); + + foreach (var vulnerability in parseResult.Vulnerabilities) + { + cancellationToken.ThrowIfCancellationRequested(); + + var state = MapState(vulnerability.AnalysisState, out var stateRaw); + var justification = MapJustification(vulnerability.AnalysisJustification); + var responses = vulnerability.AnalysisResponses; + + foreach (var affect in vulnerability.Affects) + { + var productInfo = parseResult.ResolveProduct(affect.ComponentRef); + var product = new VexProduct( + productInfo.Key, + productInfo.Name, + productInfo.Version, + productInfo.Purl, + productInfo.Cpe); + + var metadata = baseMetadata; + if (!string.IsNullOrWhiteSpace(stateRaw)) + { + metadata = metadata.SetItem("cyclonedx.analysis.state", stateRaw); + } + + if (!string.IsNullOrWhiteSpace(vulnerability.AnalysisJustification)) + { + metadata = metadata.SetItem("cyclonedx.analysis.justification", vulnerability.AnalysisJustification); + } + + if (responses.Length > 0) + { + metadata = metadata.SetItem("cyclonedx.analysis.response", string.Join(",", responses)); + } + + if (!string.IsNullOrWhiteSpace(affect.ComponentRef)) + { + metadata = metadata.SetItem("cyclonedx.affects.ref", affect.ComponentRef); + } + + var claimDocument = new VexClaimDocument( + VexDocumentFormat.CycloneDx, + document.Digest, + document.SourceUri, + parseResult.BomVersion, + signature: null); + + var claim = new VexClaim( + vulnerability.VulnerabilityId, + provider.Id, + product, + state, + claimDocument, + parseResult.FirstObserved, + parseResult.LastObserved, + justification, + vulnerability.Detail, + confidence: null, + additionalMetadata: metadata); + + claimsBuilder.Add(claim); + } + } + + var orderedClaims = claimsBuilder + .ToImmutable() + .OrderBy(static c => c.VulnerabilityId, StringComparer.Ordinal) + .ThenBy(static c => c.Product.Key, StringComparer.Ordinal) + .ToImmutableArray(); + + _logger.LogInformation( + "Normalized CycloneDX document {Source} into {ClaimCount} claim(s).", + document.SourceUri, + orderedClaims.Length); + + return ValueTask.FromResult(new VexClaimBatch( + document, + orderedClaims, + ImmutableDictionary.Empty)); + } + catch (JsonException ex) + { + _logger.LogError(ex, "Failed to parse CycloneDX VEX document {SourceUri}", document.SourceUri); + throw; + } + } + + private static VexClaimStatus MapState(string? state, out string? raw) + { + raw = state?.Trim(); + + if (!string.IsNullOrWhiteSpace(state) && StateMap.TryGetValue(state.Trim(), out var mapped)) + { + return mapped; + } + + return VexClaimStatus.UnderInvestigation; + } + + private static VexJustification? MapJustification(string? justification) + { + if (string.IsNullOrWhiteSpace(justification)) + { + return null; + } + + return JustificationMap.TryGetValue(justification.Trim(), out var mapped) + ? mapped + : null; + } + + private sealed class CycloneDxParser + { + public static CycloneDxParseResult Parse(VexRawDocument document) + { + using var json = JsonDocument.Parse(document.Content.ToArray()); + var root = json.RootElement; + + var specVersion = TryGetString(root, "specVersion"); + var bomVersion = TryGetString(root, "version"); + var serialNumber = TryGetString(root, "serialNumber"); + + var metadataTimestamp = ParseDate(TryGetProperty(root, "metadata"), "timestamp"); + var observedTimestamp = metadataTimestamp ?? document.RetrievedAt; + + var metadataBuilder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + if (!string.IsNullOrWhiteSpace(specVersion)) + { + metadataBuilder["cyclonedx.specVersion"] = specVersion!; + } + + if (!string.IsNullOrWhiteSpace(bomVersion)) + { + metadataBuilder["cyclonedx.version"] = bomVersion!; + } + + if (!string.IsNullOrWhiteSpace(serialNumber)) + { + metadataBuilder["cyclonedx.serialNumber"] = serialNumber!; + } + + var components = CollectComponents(root); + var vulnerabilities = CollectVulnerabilities(root); + + return new CycloneDxParseResult( + metadataBuilder.ToImmutable(), + bomVersion, + observedTimestamp, + observedTimestamp, + components, + vulnerabilities); + } + + private static ImmutableDictionary CollectComponents(JsonElement root) + { + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + + if (root.TryGetProperty("components", out var components) && components.ValueKind == JsonValueKind.Array) + { + foreach (var component in components.EnumerateArray()) + { + if (component.ValueKind != JsonValueKind.Object) + { + continue; + } + + var reference = TryGetString(component, "bom-ref") ?? TryGetString(component, "bomRef"); + if (string.IsNullOrWhiteSpace(reference)) + { + continue; + } + + var name = TryGetString(component, "name") ?? reference; + var version = TryGetString(component, "version"); + var purl = TryGetString(component, "purl"); + + string? cpe = null; + if (component.TryGetProperty("externalReferences", out var externalRefs) && externalRefs.ValueKind == JsonValueKind.Array) + { + foreach (var referenceEntry in externalRefs.EnumerateArray()) + { + if (referenceEntry.ValueKind != JsonValueKind.Object) + { + continue; + } + + var type = TryGetString(referenceEntry, "type"); + if (!string.Equals(type, "cpe", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (referenceEntry.TryGetProperty("url", out var url) && url.ValueKind == JsonValueKind.String) + { + cpe = url.GetString(); + break; + } + } + } + + builder[reference!] = new CycloneDxComponent(reference!, name ?? reference!, version, purl, cpe); + } + } + + return builder.ToImmutable(); + } + + private static ImmutableArray CollectVulnerabilities(JsonElement root) + { + if (!root.TryGetProperty("vulnerabilities", out var vulnerabilitiesElement) || + vulnerabilitiesElement.ValueKind != JsonValueKind.Array) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + + foreach (var vulnerability in vulnerabilitiesElement.EnumerateArray()) + { + if (vulnerability.ValueKind != JsonValueKind.Object) + { + continue; + } + + var vulnerabilityId = + TryGetString(vulnerability, "id") ?? + TryGetString(vulnerability, "bom-ref") ?? + TryGetString(vulnerability, "bomRef") ?? + TryGetString(vulnerability, "cve"); + + if (string.IsNullOrWhiteSpace(vulnerabilityId)) + { + continue; + } + + var detail = TryGetString(vulnerability, "detail") ?? TryGetString(vulnerability, "description"); + + var analysis = TryGetProperty(vulnerability, "analysis"); + var analysisState = TryGetString(analysis, "state"); + var analysisJustification = TryGetString(analysis, "justification"); + var analysisResponses = CollectResponses(analysis); + + var affects = CollectAffects(vulnerability); + if (affects.Length == 0) + { + continue; + } + + builder.Add(new CycloneDxVulnerability( + vulnerabilityId.Trim(), + detail?.Trim(), + analysisState, + analysisJustification, + analysisResponses, + affects)); + } + + return builder.ToImmutable(); + } + + private static ImmutableArray CollectResponses(JsonElement analysis) + { + if (analysis.ValueKind != JsonValueKind.Object || + !analysis.TryGetProperty("response", out var responseElement) || + responseElement.ValueKind != JsonValueKind.Array) + { + return ImmutableArray.Empty; + } + + var responses = new SortedSet(StringComparer.OrdinalIgnoreCase); + foreach (var response in responseElement.EnumerateArray()) + { + if (response.ValueKind == JsonValueKind.String) + { + var value = response.GetString(); + if (!string.IsNullOrWhiteSpace(value)) + { + responses.Add(value.Trim()); + } + } + } + + return responses.Count == 0 ? ImmutableArray.Empty : responses.ToImmutableArray(); + } + + private static ImmutableArray CollectAffects(JsonElement vulnerability) + { + if (!vulnerability.TryGetProperty("affects", out var affectsElement) || + affectsElement.ValueKind != JsonValueKind.Array) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var affect in affectsElement.EnumerateArray()) + { + if (affect.ValueKind != JsonValueKind.Object) + { + continue; + } + + var reference = TryGetString(affect, "ref"); + if (string.IsNullOrWhiteSpace(reference)) + { + continue; + } + + builder.Add(new CycloneDxAffect(reference.Trim())); + } + + return builder.ToImmutable(); + } + + private static JsonElement TryGetProperty(JsonElement element, string propertyName) + => element.ValueKind == JsonValueKind.Object && element.TryGetProperty(propertyName, out var value) + ? value + : default; + + private static string? TryGetString(JsonElement element, string propertyName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (!element.TryGetProperty(propertyName, out var value)) + { + return null; + } + + return value.ValueKind == JsonValueKind.String ? value.GetString() : null; + } + + private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) + { + var value = TryGetString(element, propertyName); + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return DateTimeOffset.TryParse(value, out var parsed) ? parsed : null; + } + } + + private sealed record CycloneDxParseResult( + ImmutableDictionary Metadata, + string? BomVersion, + DateTimeOffset FirstObserved, + DateTimeOffset LastObserved, + ImmutableDictionary Components, + ImmutableArray Vulnerabilities) + { + public CycloneDxProductInfo ResolveProduct(string? componentRef) + { + if (!string.IsNullOrWhiteSpace(componentRef) && + Components.TryGetValue(componentRef.Trim(), out var component)) + { + return new CycloneDxProductInfo(component.Reference, component.Name, component.Version, component.Purl, component.Cpe); + } + + var key = string.IsNullOrWhiteSpace(componentRef) ? "unknown-component" : componentRef.Trim(); + return new CycloneDxProductInfo(key, key, null, null, null); + } + } + + private sealed record CycloneDxComponent( + string Reference, + string Name, + string? Version, + string? Purl, + string? Cpe); + + private sealed record CycloneDxVulnerability( + string VulnerabilityId, + string? Detail, + string? AnalysisState, + string? AnalysisJustification, + ImmutableArray AnalysisResponses, + ImmutableArray Affects); + + private sealed record CycloneDxAffect(string ComponentRef); + + private sealed record CycloneDxProductInfo( + string Key, + string Name, + string? Version, + string? Purl, + string? Cpe); +} diff --git a/src/StellaOps.Vexer.Formats.CycloneDX/ServiceCollectionExtensions.cs b/src/StellaOps.Vexer.Formats.CycloneDX/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..d39c3c78 --- /dev/null +++ b/src/StellaOps.Vexer.Formats.CycloneDX/ServiceCollectionExtensions.cs @@ -0,0 +1,14 @@ +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Vexer.Core; + +namespace StellaOps.Vexer.Formats.CycloneDX; + +public static class CycloneDxFormatsServiceCollectionExtensions +{ + public static IServiceCollection AddCycloneDxNormalizer(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + services.AddSingleton(); + return services; + } +} diff --git a/src/StellaOps.Vexer.Formats.CycloneDX/StellaOps.Vexer.Formats.CycloneDX.csproj b/src/StellaOps.Vexer.Formats.CycloneDX/StellaOps.Vexer.Formats.CycloneDX.csproj new file mode 100644 index 00000000..54abf95e --- /dev/null +++ b/src/StellaOps.Vexer.Formats.CycloneDX/StellaOps.Vexer.Formats.CycloneDX.csproj @@ -0,0 +1,16 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + diff --git a/src/StellaOps.Vexer.Formats.CycloneDX/TASKS.md b/src/StellaOps.Vexer.Formats.CycloneDX/TASKS.md index 1385ef08..cf34f392 100644 --- a/src/StellaOps.Vexer.Formats.CycloneDX/TASKS.md +++ b/src/StellaOps.Vexer.Formats.CycloneDX/TASKS.md @@ -2,6 +2,6 @@ If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and # TASKS | Task | Owner(s) | Depends on | Notes | |---|---|---|---| -|VEXER-FMT-CYCLONE-01-001 – CycloneDX VEX normalizer|Team Vexer Formats|VEXER-CORE-01-001|TODO – Parse CycloneDX `analysis` data into `VexClaim` entries with deterministic component identifiers.| +|VEXER-FMT-CYCLONE-01-001 – CycloneDX VEX normalizer|Team Vexer Formats|VEXER-CORE-01-001|**DONE (2025-10-17)** – CycloneDX normalizer parses `analysis` data, resolves component references, and emits canonical `VexClaim`s; regression lives in `CycloneDxNormalizerTests`.| |VEXER-FMT-CYCLONE-01-002 – Component reference reconciliation|Team Vexer Formats|VEXER-FMT-CYCLONE-01-001|TODO – Implement helpers to reconcile component/service references against policy expectations and emit diagnostics for missing SBOM links.| |VEXER-FMT-CYCLONE-01-003 – CycloneDX export serializer|Team Vexer Formats|VEXER-EXPORT-01-001, VEXER-FMT-CYCLONE-01-001|TODO – Provide exporters producing CycloneDX VEX output with canonical ordering and hash-stable manifests.| diff --git a/src/StellaOps.Vexer.Formats.OpenVEX.Tests/OpenVexNormalizerTests.cs b/src/StellaOps.Vexer.Formats.OpenVEX.Tests/OpenVexNormalizerTests.cs new file mode 100644 index 00000000..37989d9d --- /dev/null +++ b/src/StellaOps.Vexer.Formats.OpenVEX.Tests/OpenVexNormalizerTests.cs @@ -0,0 +1,87 @@ +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Vexer.Core; +using StellaOps.Vexer.Formats.OpenVEX; + +namespace StellaOps.Vexer.Formats.OpenVEX.Tests; + +public sealed class OpenVexNormalizerTests +{ + [Fact] + public async Task NormalizeAsync_ProducesClaimsForStatements() + { + var json = """ + { + "document": { + "author": "Acme Security", + "version": "1", + "issued": "2025-10-01T00:00:00Z", + "last_updated": "2025-10-05T00:00:00Z" + }, + "statements": [ + { + "id": "statement-1", + "vulnerability": "CVE-2025-2000", + "status": "not_affected", + "justification": "code_not_present", + "products": [ + { + "id": "acme-widget@1.2.3", + "name": "Acme Widget", + "version": "1.2.3", + "purl": "pkg:acme/widget@1.2.3", + "cpe": "cpe:/a:acme:widget:1.2.3" + } + ], + "statement": "The vulnerable code was never shipped." + }, + { + "id": "statement-2", + "vulnerability": "CVE-2025-2001", + "status": "affected", + "products": [ + "pkg:acme/widget@2.0.0" + ], + "remediation": "Upgrade to 2.1.0" + } + ] + } + """; + + var rawDocument = new VexRawDocument( + "vexer:openvex", + VexDocumentFormat.OpenVex, + new Uri("https://example.com/openvex.json"), + new DateTimeOffset(2025, 10, 6, 0, 0, 0, TimeSpan.Zero), + "sha256:dummydigest", + Encoding.UTF8.GetBytes(json), + ImmutableDictionary.Empty); + + var provider = new VexProvider("vexer:openvex", "OpenVEX Provider", VexProviderKind.Vendor); + var normalizer = new OpenVexNormalizer(NullLogger.Instance); + + var batch = await normalizer.NormalizeAsync(rawDocument, provider, CancellationToken.None); + + batch.Claims.Should().HaveCount(2); + + var notAffected = batch.Claims.Single(c => c.VulnerabilityId == "CVE-2025-2000"); + notAffected.Status.Should().Be(VexClaimStatus.NotAffected); + notAffected.Justification.Should().Be(VexJustification.CodeNotPresent); + notAffected.Product.Key.Should().Be("acme-widget@1.2.3"); + notAffected.Product.Purl.Should().Be("pkg:acme/widget@1.2.3"); + notAffected.Document.Revision.Should().Be("1"); + notAffected.AdditionalMetadata["openvex.document.author"].Should().Be("Acme Security"); + notAffected.AdditionalMetadata["openvex.statement.status"].Should().Be("not_affected"); + notAffected.Detail.Should().Be("The vulnerable code was never shipped."); + + var affected = batch.Claims.Single(c => c.VulnerabilityId == "CVE-2025-2001"); + affected.Status.Should().Be(VexClaimStatus.Affected); + affected.Justification.Should().BeNull(); + affected.Product.Key.Should().Be("pkg:acme/widget@2.0.0"); + affected.Product.Name.Should().Be("pkg:acme/widget@2.0.0"); + affected.Detail.Should().Be("Upgrade to 2.1.0"); + } +} diff --git a/src/StellaOps.Vexer.Formats.OpenVEX.Tests/StellaOps.Vexer.Formats.OpenVEX.Tests.csproj b/src/StellaOps.Vexer.Formats.OpenVEX.Tests/StellaOps.Vexer.Formats.OpenVEX.Tests.csproj new file mode 100644 index 00000000..bdd68e59 --- /dev/null +++ b/src/StellaOps.Vexer.Formats.OpenVEX.Tests/StellaOps.Vexer.Formats.OpenVEX.Tests.csproj @@ -0,0 +1,17 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Formats.OpenVEX/OpenVexNormalizer.cs b/src/StellaOps.Vexer.Formats.OpenVEX/OpenVexNormalizer.cs new file mode 100644 index 00000000..ac748af9 --- /dev/null +++ b/src/StellaOps.Vexer.Formats.OpenVEX/OpenVexNormalizer.cs @@ -0,0 +1,367 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Vexer.Core; + +namespace StellaOps.Vexer.Formats.OpenVEX; + +public sealed class OpenVexNormalizer : IVexNormalizer +{ + private static readonly ImmutableDictionary StatusMap = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["affected"] = VexClaimStatus.Affected, + ["not_affected"] = VexClaimStatus.NotAffected, + ["fixed"] = VexClaimStatus.Fixed, + ["under_investigation"] = VexClaimStatus.UnderInvestigation, + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + private static readonly ImmutableDictionary JustificationMap = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["component_not_present"] = VexJustification.ComponentNotPresent, + ["component_not_configured"] = VexJustification.ComponentNotConfigured, + ["vulnerable_code_not_present"] = VexJustification.VulnerableCodeNotPresent, + ["vulnerable_code_not_in_execute_path"] = VexJustification.VulnerableCodeNotInExecutePath, + ["vulnerable_code_cannot_be_controlled_by_adversary"] = VexJustification.VulnerableCodeCannotBeControlledByAdversary, + ["inline_mitigations_already_exist"] = VexJustification.InlineMitigationsAlreadyExist, + ["protected_by_mitigating_control"] = VexJustification.ProtectedByMitigatingControl, + ["protected_by_compensating_control"] = VexJustification.ProtectedByCompensatingControl, + ["code_not_present"] = VexJustification.CodeNotPresent, + ["code_not_reachable"] = VexJustification.CodeNotReachable, + ["requires_configuration"] = VexJustification.RequiresConfiguration, + ["requires_dependency"] = VexJustification.RequiresDependency, + ["requires_environment"] = VexJustification.RequiresEnvironment, + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + private readonly ILogger _logger; + + public OpenVexNormalizer(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string Format => VexDocumentFormat.OpenVex.ToString().ToLowerInvariant(); + + public bool CanHandle(VexRawDocument document) + => document is not null && document.Format == VexDocumentFormat.OpenVex; + + public ValueTask NormalizeAsync(VexRawDocument document, VexProvider provider, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(provider); + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var result = OpenVexParser.Parse(document); + var claims = ImmutableArray.CreateBuilder(result.Statements.Length); + + foreach (var statement in result.Statements) + { + cancellationToken.ThrowIfCancellationRequested(); + + var status = MapStatus(statement.Status); + var justification = MapJustification(statement.Justification); + + foreach (var product in statement.Products) + { + var vexProduct = new VexProduct( + product.Key, + product.Name, + product.Version, + product.Purl, + product.Cpe); + + var metadata = result.Metadata; + + metadata = metadata.SetItem("openvex.statement.id", statement.Id); + if (!string.IsNullOrWhiteSpace(statement.Status)) + { + metadata = metadata.SetItem("openvex.statement.status", statement.Status!); + } + + if (!string.IsNullOrWhiteSpace(statement.Justification)) + { + metadata = metadata.SetItem("openvex.statement.justification", statement.Justification!); + } + + if (!string.IsNullOrWhiteSpace(product.OriginalId)) + { + metadata = metadata.SetItem("openvex.product.source", product.OriginalId!); + } + + var claimDocument = new VexClaimDocument( + VexDocumentFormat.OpenVex, + document.Digest, + document.SourceUri, + result.DocumentVersion, + signature: null); + + var claim = new VexClaim( + statement.Vulnerability, + provider.Id, + vexProduct, + status, + claimDocument, + result.FirstObserved, + result.LastObserved, + justification, + statement.Remarks, + confidence: null, + additionalMetadata: metadata); + + claims.Add(claim); + } + } + + var orderedClaims = claims + .ToImmutable() + .OrderBy(static claim => claim.VulnerabilityId, StringComparer.Ordinal) + .ThenBy(static claim => claim.Product.Key, StringComparer.Ordinal) + .ToImmutableArray(); + + _logger.LogInformation( + "Normalized OpenVEX document {Source} into {ClaimCount} claim(s).", + document.SourceUri, + orderedClaims.Length); + + return ValueTask.FromResult(new VexClaimBatch( + document, + orderedClaims, + ImmutableDictionary.Empty)); + } + catch (JsonException ex) + { + _logger.LogError(ex, "Failed to parse OpenVEX document {SourceUri}", document.SourceUri); + throw; + } + } + + private static VexClaimStatus MapStatus(string? status) + { + if (!string.IsNullOrWhiteSpace(status) && StatusMap.TryGetValue(status.Trim(), out var mapped)) + { + return mapped; + } + + return VexClaimStatus.UnderInvestigation; + } + + private static VexJustification? MapJustification(string? justification) + { + if (string.IsNullOrWhiteSpace(justification)) + { + return null; + } + + return JustificationMap.TryGetValue(justification.Trim(), out var mapped) + ? mapped + : null; + } + + private static class OpenVexParser + { + public static OpenVexParseResult Parse(VexRawDocument document) + { + using var json = JsonDocument.Parse(document.Content.ToArray()); + var root = json.RootElement; + + var metadata = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + + var documentElement = TryGetProperty(root, "document"); + var version = TryGetString(documentElement, "version"); + var author = TryGetString(documentElement, "author"); + + if (!string.IsNullOrWhiteSpace(version)) + { + metadata["openvex.document.version"] = version!; + } + + if (!string.IsNullOrWhiteSpace(author)) + { + metadata["openvex.document.author"] = author!; + } + + var issued = ParseDate(documentElement, "issued"); + var lastUpdated = ParseDate(documentElement, "last_updated") ?? issued ?? document.RetrievedAt; + var effectiveDate = ParseDate(documentElement, "effective_date") ?? issued ?? document.RetrievedAt; + + var statements = CollectStatements(root); + + return new OpenVexParseResult( + metadata.ToImmutable(), + version, + effectiveDate, + lastUpdated, + statements); + } + + private static ImmutableArray CollectStatements(JsonElement root) + { + if (!root.TryGetProperty("statements", out var statementsElement) || + statementsElement.ValueKind != JsonValueKind.Array) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var statement in statementsElement.EnumerateArray()) + { + if (statement.ValueKind != JsonValueKind.Object) + { + continue; + } + + var vulnerability = TryGetString(statement, "vulnerability") ?? TryGetString(statement, "vuln") ?? string.Empty; + if (string.IsNullOrWhiteSpace(vulnerability)) + { + continue; + } + + var id = TryGetString(statement, "id") ?? Guid.NewGuid().ToString(); + var status = TryGetString(statement, "status"); + var justification = TryGetString(statement, "justification"); + var remarks = TryGetString(statement, "remediation") ?? TryGetString(statement, "statement"); + var products = CollectProducts(statement); + if (products.Length == 0) + { + continue; + } + + builder.Add(new OpenVexStatement( + id, + vulnerability.Trim(), + status, + justification, + remarks, + products)); + } + + return builder.ToImmutable(); + } + + private static ImmutableArray CollectProducts(JsonElement statement) + { + if (!statement.TryGetProperty("products", out var productsElement) || + productsElement.ValueKind != JsonValueKind.Array) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var product in productsElement.EnumerateArray()) + { + if (product.ValueKind != JsonValueKind.String && product.ValueKind != JsonValueKind.Object) + { + continue; + } + + if (product.ValueKind == JsonValueKind.String) + { + var value = product.GetString(); + if (string.IsNullOrWhiteSpace(value)) + { + continue; + } + + builder.Add(OpenVexProduct.FromString(value.Trim())); + continue; + } + + var id = TryGetString(product, "id") ?? TryGetString(product, "product_id"); + var name = TryGetString(product, "name"); + var version = TryGetString(product, "version"); + var purl = TryGetString(product, "purl"); + var cpe = TryGetString(product, "cpe"); + + if (string.IsNullOrWhiteSpace(id) && string.IsNullOrWhiteSpace(purl)) + { + continue; + } + + builder.Add(new OpenVexProduct( + id ?? purl!, + name ?? id ?? purl!, + version, + purl, + cpe, + OriginalId: id)); + } + + return builder.ToImmutable(); + } + + private static JsonElement TryGetProperty(JsonElement element, string propertyName) + => element.ValueKind == JsonValueKind.Object && element.TryGetProperty(propertyName, out var value) + ? value + : default; + + private static string? TryGetString(JsonElement element, string propertyName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (!element.TryGetProperty(propertyName, out var value)) + { + return null; + } + + return value.ValueKind == JsonValueKind.String ? value.GetString() : null; + } + + private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) + { + var value = TryGetString(element, propertyName); + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return DateTimeOffset.TryParse(value, out var parsed) ? parsed : null; + } + } + + private sealed record OpenVexParseResult( + ImmutableDictionary Metadata, + string? DocumentVersion, + DateTimeOffset FirstObserved, + DateTimeOffset LastObserved, + ImmutableArray Statements); + + private sealed record OpenVexStatement( + string Id, + string Vulnerability, + string? Status, + string? Justification, + string? Remarks, + ImmutableArray Products); + + private sealed record OpenVexProduct( + string Key, + string Name, + string? Version, + string? Purl, + string? Cpe, + string? OriginalId) + { + public static OpenVexProduct FromString(string value) + { + var key = value; + string? purl = null; + string? name = value; + + if (value.StartsWith("pkg:", StringComparison.OrdinalIgnoreCase)) + { + purl = value; + } + + return new OpenVexProduct(key, name, null, purl, null, OriginalId: value); + } + } +} diff --git a/src/StellaOps.Vexer.Formats.OpenVEX/ServiceCollectionExtensions.cs b/src/StellaOps.Vexer.Formats.OpenVEX/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..5866009e --- /dev/null +++ b/src/StellaOps.Vexer.Formats.OpenVEX/ServiceCollectionExtensions.cs @@ -0,0 +1,14 @@ +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Vexer.Core; + +namespace StellaOps.Vexer.Formats.OpenVEX; + +public static class OpenVexFormatsServiceCollectionExtensions +{ + public static IServiceCollection AddOpenVexNormalizer(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + services.AddSingleton(); + return services; + } +} diff --git a/src/StellaOps.Vexer.Formats.OpenVEX/StellaOps.Vexer.Formats.OpenVEX.csproj b/src/StellaOps.Vexer.Formats.OpenVEX/StellaOps.Vexer.Formats.OpenVEX.csproj new file mode 100644 index 00000000..54abf95e --- /dev/null +++ b/src/StellaOps.Vexer.Formats.OpenVEX/StellaOps.Vexer.Formats.OpenVEX.csproj @@ -0,0 +1,16 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + diff --git a/src/StellaOps.Vexer.Formats.OpenVEX/TASKS.md b/src/StellaOps.Vexer.Formats.OpenVEX/TASKS.md index 63991180..75210974 100644 --- a/src/StellaOps.Vexer.Formats.OpenVEX/TASKS.md +++ b/src/StellaOps.Vexer.Formats.OpenVEX/TASKS.md @@ -2,6 +2,6 @@ If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and # TASKS | Task | Owner(s) | Depends on | Notes | |---|---|---|---| -|VEXER-FMT-OPENVEX-01-001 – OpenVEX normalizer|Team Vexer Formats|VEXER-CORE-01-001|TODO – Implement OpenVEX parser covering statements, products, and `status/justification` mapping with provenance metadata.| +|VEXER-FMT-OPENVEX-01-001 – OpenVEX normalizer|Team Vexer Formats|VEXER-CORE-01-001|**DONE (2025-10-17)** – OpenVEX normalizer parses statements/products, maps status/justification, and surfaces provenance metadata; coverage in `OpenVexNormalizerTests`.| |VEXER-FMT-OPENVEX-01-002 – Statement merge utilities|Team Vexer Formats|VEXER-FMT-OPENVEX-01-001|TODO – Add reducers merging multiple OpenVEX statements, resolving conflicts deterministically, and emitting policy diagnostics.| |VEXER-FMT-OPENVEX-01-003 – OpenVEX export writer|Team Vexer Formats|VEXER-EXPORT-01-001, VEXER-FMT-OPENVEX-01-001|TODO – Provide export serializer generating canonical OpenVEX documents with optional SBOM references and hash-stable ordering.| diff --git a/src/StellaOps.Vexer.Storage.Mongo/IVexStorageContracts.cs b/src/StellaOps.Vexer.Storage.Mongo/IVexStorageContracts.cs index 32755feb..7009f034 100644 --- a/src/StellaOps.Vexer.Storage.Mongo/IVexStorageContracts.cs +++ b/src/StellaOps.Vexer.Storage.Mongo/IVexStorageContracts.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.Threading; using System.Threading.Tasks; using StellaOps.Vexer.Core; @@ -24,6 +25,18 @@ public interface IVexConsensusStore ValueTask SaveAsync(VexConsensus consensus, CancellationToken cancellationToken); } +public sealed record VexConnectorState( + string ConnectorId, + DateTimeOffset? LastUpdated, + ImmutableArray DocumentDigests); + +public interface IVexConnectorStateRepository +{ + ValueTask GetAsync(string connectorId, CancellationToken cancellationToken); + + ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken); +} + public interface IVexCacheIndex { ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken); diff --git a/src/StellaOps.Vexer.Storage.Mongo/MongoVexConnectorStateRepository.cs b/src/StellaOps.Vexer.Storage.Mongo/MongoVexConnectorStateRepository.cs new file mode 100644 index 00000000..b3dda181 --- /dev/null +++ b/src/StellaOps.Vexer.Storage.Mongo/MongoVexConnectorStateRepository.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Driver; + +namespace StellaOps.Vexer.Storage.Mongo; + +public sealed class MongoVexConnectorStateRepository : IVexConnectorStateRepository +{ + private readonly IMongoCollection _collection; + + public MongoVexConnectorStateRepository(IMongoDatabase database) + { + ArgumentNullException.ThrowIfNull(database); + VexMongoMappingRegistry.Register(); + _collection = database.GetCollection(VexMongoCollectionNames.ConnectorState); + } + + public async ValueTask GetAsync(string connectorId, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectorId); + + var filter = Builders.Filter.Eq(x => x.ConnectorId, connectorId.Trim()); + var document = await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToRecord(); + } + + public async ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(state); + + var document = VexConnectorStateDocument.FromRecord(state.WithNormalizedDigests()); + var filter = Builders.Filter.Eq(x => x.ConnectorId, document.ConnectorId); + await _collection.ReplaceOneAsync(filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); + } +} + +internal static class VexConnectorStateExtensions +{ + private const int MaxDigestHistory = 200; + + public static VexConnectorState WithNormalizedDigests(this VexConnectorState state) + { + var digests = state.DocumentDigests; + if (digests.Length <= MaxDigestHistory) + { + return state; + } + + var trimmed = digests.Skip(digests.Length - MaxDigestHistory).ToImmutableArray(); + return state with { DocumentDigests = trimmed }; + } +} diff --git a/src/StellaOps.Vexer.Storage.Mongo/ServiceCollectionExtensions.cs b/src/StellaOps.Vexer.Storage.Mongo/ServiceCollectionExtensions.cs index 1a22c49a..984bcdc3 100644 --- a/src/StellaOps.Vexer.Storage.Mongo/ServiceCollectionExtensions.cs +++ b/src/StellaOps.Vexer.Storage.Mongo/ServiceCollectionExtensions.cs @@ -16,6 +16,7 @@ public static class VexMongoServiceCollectionExtensions services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddHostedService(); diff --git a/src/StellaOps.Vexer.Storage.Mongo/VexMongoMappingRegistry.cs b/src/StellaOps.Vexer.Storage.Mongo/VexMongoMappingRegistry.cs index 4106d9dd..a36abdc5 100644 --- a/src/StellaOps.Vexer.Storage.Mongo/VexMongoMappingRegistry.cs +++ b/src/StellaOps.Vexer.Storage.Mongo/VexMongoMappingRegistry.cs @@ -39,6 +39,7 @@ public static class VexMongoMappingRegistry RegisterClassMap(); RegisterClassMap(); RegisterClassMap(); + RegisterClassMap(); } private static void RegisterClassMap() @@ -66,4 +67,5 @@ public static class VexMongoCollectionNames public const string Consensus = "vex.consensus"; public const string Exports = "vex.exports"; public const string Cache = "vex.cache"; + public const string ConnectorState = "vex.connector_state"; } diff --git a/src/StellaOps.Vexer.Storage.Mongo/VexMongoModels.cs b/src/StellaOps.Vexer.Storage.Mongo/VexMongoModels.cs index 8e106f99..c9f61f48 100644 --- a/src/StellaOps.Vexer.Storage.Mongo/VexMongoModels.cs +++ b/src/StellaOps.Vexer.Storage.Mongo/VexMongoModels.cs @@ -570,3 +570,35 @@ internal sealed class VexCacheEntryRecord expires); } } + +[BsonIgnoreExtraElements] +internal sealed class VexConnectorStateDocument +{ + [BsonId] + public string ConnectorId { get; set; } = default!; + + public DateTime? LastUpdated { get; set; } + = null; + + public List DocumentDigests { get; set; } = new(); + + public static VexConnectorStateDocument FromRecord(VexConnectorState state) + => new() + { + ConnectorId = state.ConnectorId, + LastUpdated = state.LastUpdated?.UtcDateTime, + DocumentDigests = state.DocumentDigests.ToList(), + }; + + public VexConnectorState ToRecord() + { + var lastUpdated = LastUpdated.HasValue + ? new DateTimeOffset(DateTime.SpecifyKind(LastUpdated.Value, DateTimeKind.Utc)) + : (DateTimeOffset?)null; + + return new VexConnectorState( + ConnectorId, + lastUpdated, + DocumentDigests.ToImmutableArray()); + } +} diff --git a/src/StellaOps.Vexer.WebService/Program.cs b/src/StellaOps.Vexer.WebService/Program.cs index 30a2063d..cef66a25 100644 --- a/src/StellaOps.Vexer.WebService/Program.cs +++ b/src/StellaOps.Vexer.WebService/Program.cs @@ -7,6 +7,7 @@ using StellaOps.Vexer.Attestation.Transparency; using StellaOps.Vexer.ArtifactStores.S3.Extensions; using StellaOps.Vexer.Export; using StellaOps.Vexer.Storage.Mongo; +using StellaOps.Vexer.Connectors.RedHat.CSAF.DependencyInjection; var builder = WebApplication.CreateBuilder(args); var configuration = builder.Configuration; @@ -21,6 +22,7 @@ services.AddVexExportEngine(); services.AddVexExportCacheServices(); services.AddVexAttestation(); services.Configure(configuration.GetSection("Vexer:Attestation:Client")); +services.AddRedHatCsafConnector(); var rekorSection = configuration.GetSection("Vexer:Attestation:Rekor"); if (rekorSection.Exists()) diff --git a/src/StellaOps.Vexer.WebService/TASKS.md b/src/StellaOps.Vexer.WebService/TASKS.md index 8829da26..f29650c5 100644 --- a/src/StellaOps.Vexer.WebService/TASKS.md +++ b/src/StellaOps.Vexer.WebService/TASKS.md @@ -2,7 +2,7 @@ If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and # TASKS | Task | Owner(s) | Depends on | Notes | |---|---|---|---| -|VEXER-WEB-01-001 – Minimal API bootstrap & DI|Team Vexer WebService|VEXER-CORE-01-003, VEXER-STORAGE-01-003|TODO – Scaffold ASP.NET host, register connectors/normalizers via plugin loader, bind policy/storage/attestation services, and expose `/vexer/status`.| +|VEXER-WEB-01-001 – Minimal API bootstrap & DI|Team Vexer WebService|VEXER-CORE-01-003, VEXER-STORAGE-01-003|**DONE (2025-10-17)** – Minimal API host composes storage/export/attestation/artifact stores, binds Mongo/attestation options, and exposes `/vexer/status` + health endpoints with regression coverage in `StatusEndpointTests`.| |VEXER-WEB-01-002 – Ingest & reconcile endpoints|Team Vexer WebService|VEXER-WEB-01-001|TODO – Implement `/vexer/init`, `/vexer/ingest/run`, `/vexer/ingest/resume`, `/vexer/reconcile` with token scope enforcement and structured run telemetry.| |VEXER-WEB-01-003 – Export & verify endpoints|Team Vexer WebService|VEXER-WEB-01-001, VEXER-EXPORT-01-001, VEXER-ATTEST-01-001|TODO – Add `/vexer/export`, `/vexer/export/{id}`, `/vexer/export/{id}/download`, `/vexer/verify`, returning artifact + attestation metadata with cache awareness.| |VEXER-WEB-01-004 – Resolve API & signed responses|Team Vexer WebService|VEXER-WEB-01-001, VEXER-ATTEST-01-002|TODO – Deliver `/vexer/resolve` (subject/context), return consensus + score envelopes, attach cosign/Rekor metadata, and document auth + rate guardrails.| diff --git a/src/StellaOps.Vexer.Worker.Tests/StellaOps.Vexer.Worker.Tests.csproj b/src/StellaOps.Vexer.Worker.Tests/StellaOps.Vexer.Worker.Tests.csproj new file mode 100644 index 00000000..9e3c17ca --- /dev/null +++ b/src/StellaOps.Vexer.Worker.Tests/StellaOps.Vexer.Worker.Tests.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Worker.Tests/VexWorkerOptionsTests.cs b/src/StellaOps.Vexer.Worker.Tests/VexWorkerOptionsTests.cs new file mode 100644 index 00000000..8860638c --- /dev/null +++ b/src/StellaOps.Vexer.Worker.Tests/VexWorkerOptionsTests.cs @@ -0,0 +1,77 @@ +using FluentAssertions; +using StellaOps.Vexer.Worker.Options; +using StellaOps.Vexer.Worker.Scheduling; +using Xunit; + +namespace StellaOps.Vexer.Worker.Tests; + +public sealed class VexWorkerOptionsTests +{ + [Fact] + public void ResolveSchedules_UsesDefaultIntervalWhenNotSpecified() + { + var options = new VexWorkerOptions + { + DefaultInterval = TimeSpan.FromMinutes(30), + OfflineInterval = TimeSpan.FromHours(6), + OfflineMode = false, + }; + options.Providers.Add(new VexWorkerProviderOptions { ProviderId = "vexer:redhat" }); + + var schedules = options.ResolveSchedules(); + + schedules.Should().ContainSingle(); + schedules[0].Interval.Should().Be(TimeSpan.FromMinutes(30)); + } + + [Fact] + public void ResolveSchedules_HonorsOfflineInterval() + { + var options = new VexWorkerOptions + { + DefaultInterval = TimeSpan.FromMinutes(30), + OfflineInterval = TimeSpan.FromHours(8), + OfflineMode = true, + }; + options.Providers.Add(new VexWorkerProviderOptions { ProviderId = "vexer:offline" }); + + var schedules = options.ResolveSchedules(); + + schedules.Should().ContainSingle(); + schedules[0].Interval.Should().Be(TimeSpan.FromHours(8)); + } + + [Fact] + public void ResolveSchedules_SkipsDisabledProviders() + { + var options = new VexWorkerOptions(); + options.Providers.Add(new VexWorkerProviderOptions { ProviderId = "vexer:enabled" }); + options.Providers.Add(new VexWorkerProviderOptions { ProviderId = "vexer:disabled", Enabled = false }); + + var schedules = options.ResolveSchedules(); + + schedules.Should().HaveCount(1); + schedules[0].ProviderId.Should().Be("vexer:enabled"); + } + + [Fact] + public void ResolveSchedules_UsesProviderIntervalOverride() + { + var options = new VexWorkerOptions + { + DefaultInterval = TimeSpan.FromMinutes(15), + }; + options.Providers.Add(new VexWorkerProviderOptions + { + ProviderId = "vexer:custom", + Interval = TimeSpan.FromMinutes(5), + InitialDelay = TimeSpan.FromSeconds(10), + }); + + var schedules = options.ResolveSchedules(); + + schedules.Should().ContainSingle(); + schedules[0].Interval.Should().Be(TimeSpan.FromMinutes(5)); + schedules[0].InitialDelay.Should().Be(TimeSpan.FromSeconds(10)); + } +} diff --git a/src/StellaOps.Vexer.Worker/Options/VexWorkerOptions.cs b/src/StellaOps.Vexer.Worker/Options/VexWorkerOptions.cs new file mode 100644 index 00000000..e8a79079 --- /dev/null +++ b/src/StellaOps.Vexer.Worker/Options/VexWorkerOptions.cs @@ -0,0 +1,62 @@ +using System.Collections.Generic; +using StellaOps.Vexer.Worker.Scheduling; + +namespace StellaOps.Vexer.Worker.Options; + +public sealed class VexWorkerOptions +{ + public TimeSpan DefaultInterval { get; set; } = TimeSpan.FromHours(1); + + public TimeSpan OfflineInterval { get; set; } = TimeSpan.FromHours(6); + + public TimeSpan DefaultInitialDelay { get; set; } = TimeSpan.FromMinutes(5); + + public bool OfflineMode { get; set; } + + public IList Providers { get; } = new List(); + + internal IReadOnlyList ResolveSchedules() + { + var schedules = new List(); + foreach (var provider in Providers) + { + if (!provider.Enabled) + { + continue; + } + + var providerId = provider.ProviderId?.Trim(); + if (string.IsNullOrWhiteSpace(providerId)) + { + continue; + } + + var interval = provider.Interval ?? (OfflineMode ? OfflineInterval : DefaultInterval); + if (interval <= TimeSpan.Zero) + { + continue; + } + + var initialDelay = provider.InitialDelay ?? DefaultInitialDelay; + if (initialDelay < TimeSpan.Zero) + { + initialDelay = TimeSpan.Zero; + } + + schedules.Add(new VexWorkerSchedule(providerId, interval, initialDelay)); + } + + return schedules; + } +} + +public sealed class VexWorkerProviderOptions +{ + public string ProviderId { get; set; } = string.Empty; + + public bool Enabled { get; set; } = true; + + public TimeSpan? Interval { get; set; } + + public TimeSpan? InitialDelay { get; set; } +} diff --git a/src/StellaOps.Vexer.Worker/Options/VexWorkerOptionsValidator.cs b/src/StellaOps.Vexer.Worker/Options/VexWorkerOptionsValidator.cs new file mode 100644 index 00000000..bf53b771 --- /dev/null +++ b/src/StellaOps.Vexer.Worker/Options/VexWorkerOptionsValidator.cs @@ -0,0 +1,50 @@ +using System.Collections.Generic; +using Microsoft.Extensions.Options; + +namespace StellaOps.Vexer.Worker.Options; + +internal sealed class VexWorkerOptionsValidator : IValidateOptions +{ + public ValidateOptionsResult Validate(string? name, VexWorkerOptions options) + { + var failures = new List(); + + if (options.DefaultInterval <= TimeSpan.Zero) + { + failures.Add("Vexer.Worker.DefaultInterval must be greater than zero."); + } + + if (options.OfflineInterval <= TimeSpan.Zero) + { + failures.Add("Vexer.Worker.OfflineInterval must be greater than zero."); + } + + if (options.DefaultInitialDelay < TimeSpan.Zero) + { + failures.Add("Vexer.Worker.DefaultInitialDelay cannot be negative."); + } + + for (var i = 0; i < options.Providers.Count; i++) + { + var provider = options.Providers[i]; + if (string.IsNullOrWhiteSpace(provider.ProviderId)) + { + failures.Add($"Vexer.Worker.Providers[{i}].ProviderId must be set."); + } + + if (provider.Interval is { } interval && interval <= TimeSpan.Zero) + { + failures.Add($"Vexer.Worker.Providers[{i}].Interval must be greater than zero when specified."); + } + + if (provider.InitialDelay is { } delay && delay < TimeSpan.Zero) + { + failures.Add($"Vexer.Worker.Providers[{i}].InitialDelay cannot be negative."); + } + } + + return failures.Count > 0 + ? ValidateOptionsResult.Fail(failures) + : ValidateOptionsResult.Success; + } +} diff --git a/src/StellaOps.Vexer.Worker/Options/VexWorkerPluginOptions.cs b/src/StellaOps.Vexer.Worker/Options/VexWorkerPluginOptions.cs new file mode 100644 index 00000000..487cd448 --- /dev/null +++ b/src/StellaOps.Vexer.Worker/Options/VexWorkerPluginOptions.cs @@ -0,0 +1,21 @@ +using System; +using System.IO; + +namespace StellaOps.Vexer.Worker.Options; + +public sealed class VexWorkerPluginOptions +{ + public string? Directory { get; set; } + + public string? SearchPattern { get; set; } + + internal string ResolveDirectory() + => string.IsNullOrWhiteSpace(Directory) + ? Path.Combine(AppContext.BaseDirectory, "plugins") + : Path.GetFullPath(Directory); + + internal string ResolveSearchPattern() + => string.IsNullOrWhiteSpace(SearchPattern) + ? "StellaOps.Vexer.Connectors.*.dll" + : SearchPattern!; +} diff --git a/src/StellaOps.Vexer.Worker/Program.cs b/src/StellaOps.Vexer.Worker/Program.cs new file mode 100644 index 00000000..247c4f39 --- /dev/null +++ b/src/StellaOps.Vexer.Worker/Program.cs @@ -0,0 +1,59 @@ +using System.IO; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Plugin; +using StellaOps.Vexer.Connectors.RedHat.CSAF.DependencyInjection; +using StellaOps.Vexer.Worker.Options; +using StellaOps.Vexer.Worker.Scheduling; + +var builder = Host.CreateApplicationBuilder(args); +var services = builder.Services; +var configuration = builder.Configuration; +services.AddOptions() + .Bind(configuration.GetSection("Vexer:Worker")) + .ValidateOnStart(); + +services.Configure(configuration.GetSection("Vexer:Worker:Plugins")); +services.AddRedHatCsafConnector(); + +services.AddSingleton, VexWorkerOptionsValidator>(); +services.AddSingleton(TimeProvider.System); +services.PostConfigure(options => +{ + if (!options.Providers.Any(provider => string.Equals(provider.ProviderId, "vexer:redhat", StringComparison.OrdinalIgnoreCase))) + { + options.Providers.Add(new VexWorkerProviderOptions + { + ProviderId = "vexer:redhat", + }); + } +}); +services.AddSingleton(provider => +{ + var pluginOptions = provider.GetRequiredService>().Value; + var catalog = new PluginCatalog(); + + var directory = pluginOptions.ResolveDirectory(); + if (Directory.Exists(directory)) + { + catalog.AddFromDirectory(directory, pluginOptions.ResolveSearchPattern()); + } + else + { + var logger = provider.GetRequiredService>(); + logger.LogWarning("Vexer worker plugin directory '{Directory}' does not exist; proceeding without external connectors.", directory); + } + + return catalog; +}); + +services.AddSingleton(); +services.AddHostedService(); + +var host = builder.Build(); +await host.RunAsync(); + +public partial class Program; diff --git a/src/StellaOps.Vexer.Worker/Properties/AssemblyInfo.cs b/src/StellaOps.Vexer.Worker/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..bbbd22fd --- /dev/null +++ b/src/StellaOps.Vexer.Worker/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Vexer.Worker.Tests")] diff --git a/src/StellaOps.Vexer.Worker/Scheduling/DefaultVexProviderRunner.cs b/src/StellaOps.Vexer.Worker/Scheduling/DefaultVexProviderRunner.cs new file mode 100644 index 00000000..113d1f7c --- /dev/null +++ b/src/StellaOps.Vexer.Worker/Scheduling/DefaultVexProviderRunner.cs @@ -0,0 +1,47 @@ +using System; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.Plugin; + +namespace StellaOps.Vexer.Worker.Scheduling; + +internal sealed class DefaultVexProviderRunner : IVexProviderRunner +{ + private readonly IServiceProvider _serviceProvider; + private readonly PluginCatalog _pluginCatalog; + private readonly ILogger _logger; + + public DefaultVexProviderRunner( + IServiceProvider serviceProvider, + PluginCatalog pluginCatalog, + ILogger logger) + { + _serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); + _pluginCatalog = pluginCatalog ?? throw new ArgumentNullException(nameof(pluginCatalog)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public ValueTask RunAsync(string providerId, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(providerId); + + using var scope = _serviceProvider.CreateScope(); + var availablePlugins = _pluginCatalog.GetAvailableConnectorPlugins(scope.ServiceProvider); + var matched = availablePlugins.FirstOrDefault(plugin => + string.Equals(plugin.Name, providerId, StringComparison.OrdinalIgnoreCase)); + + if (matched is null) + { + _logger.LogInformation("No connector plugin registered for provider {ProviderId}; nothing to execute.", providerId); + return ValueTask.CompletedTask; + } + + _logger.LogInformation( + "Connector plugin {PluginName} ({ProviderId}) is available. Execution hooks will be added in subsequent tasks.", + matched.Name, + providerId); + + return ValueTask.CompletedTask; + } +} diff --git a/src/StellaOps.Vexer.Worker/Scheduling/IVexProviderRunner.cs b/src/StellaOps.Vexer.Worker/Scheduling/IVexProviderRunner.cs new file mode 100644 index 00000000..8d20f39d --- /dev/null +++ b/src/StellaOps.Vexer.Worker/Scheduling/IVexProviderRunner.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Vexer.Worker.Scheduling; + +internal interface IVexProviderRunner +{ + ValueTask RunAsync(string providerId, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Vexer.Worker/Scheduling/VexWorkerHostedService.cs b/src/StellaOps.Vexer.Worker/Scheduling/VexWorkerHostedService.cs new file mode 100644 index 00000000..b37bbe38 --- /dev/null +++ b/src/StellaOps.Vexer.Worker/Scheduling/VexWorkerHostedService.cs @@ -0,0 +1,110 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Vexer.Worker.Options; + +namespace StellaOps.Vexer.Worker.Scheduling; + +internal sealed class VexWorkerHostedService : BackgroundService +{ + private readonly IOptions _options; + private readonly IVexProviderRunner _runner; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + + public VexWorkerHostedService( + IOptions options, + IVexProviderRunner runner, + ILogger logger, + TimeProvider timeProvider) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _runner = runner ?? throw new ArgumentNullException(nameof(runner)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + var schedules = _options.Value.ResolveSchedules(); + if (schedules.Count == 0) + { + _logger.LogWarning("Vexer worker has no configured provider schedules; the service will remain idle."); + await Task.CompletedTask; + return; + } + + _logger.LogInformation("Vexer worker starting with {ProviderCount} provider schedule(s).", schedules.Count); + + var tasks = new List(schedules.Count); + foreach (var schedule in schedules) + { + tasks.Add(RunScheduleAsync(schedule, stoppingToken)); + } + + await Task.WhenAll(tasks); + } + + private async Task RunScheduleAsync(VexWorkerSchedule schedule, CancellationToken cancellationToken) + { + try + { + if (schedule.InitialDelay > TimeSpan.Zero) + { + _logger.LogInformation( + "Provider {ProviderId} initial delay of {InitialDelay} before first execution.", + schedule.ProviderId, + schedule.InitialDelay); + + await Task.Delay(schedule.InitialDelay, cancellationToken).ConfigureAwait(false); + } + + using var timer = new PeriodicTimer(schedule.Interval); + do + { + var startedAt = _timeProvider.GetUtcNow(); + _logger.LogInformation( + "Provider {ProviderId} run started at {StartedAt}. Interval={Interval}.", + schedule.ProviderId, + startedAt, + schedule.Interval); + + try + { + await _runner.RunAsync(schedule.ProviderId, cancellationToken).ConfigureAwait(false); + + var completedAt = _timeProvider.GetUtcNow(); + var elapsed = completedAt - startedAt; + + _logger.LogInformation( + "Provider {ProviderId} run completed at {CompletedAt} (duration {Duration}).", + schedule.ProviderId, + completedAt, + elapsed); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + _logger.LogInformation("Provider {ProviderId} run cancelled.", schedule.ProviderId); + break; + } + catch (Exception ex) + { + _logger.LogError( + ex, + "Provider {ProviderId} run failed: {Message}", + schedule.ProviderId, + ex.Message); + } + } + while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false)); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + _logger.LogInformation("Provider {ProviderId} schedule cancelled.", schedule.ProviderId); + } + } +} diff --git a/src/StellaOps.Vexer.Worker/Scheduling/VexWorkerSchedule.cs b/src/StellaOps.Vexer.Worker/Scheduling/VexWorkerSchedule.cs new file mode 100644 index 00000000..81b2c68a --- /dev/null +++ b/src/StellaOps.Vexer.Worker/Scheduling/VexWorkerSchedule.cs @@ -0,0 +1,3 @@ +namespace StellaOps.Vexer.Worker.Scheduling; + +internal sealed record VexWorkerSchedule(string ProviderId, TimeSpan Interval, TimeSpan InitialDelay); diff --git a/src/StellaOps.Vexer.Worker/StellaOps.Vexer.Worker.csproj b/src/StellaOps.Vexer.Worker/StellaOps.Vexer.Worker.csproj new file mode 100644 index 00000000..93a50200 --- /dev/null +++ b/src/StellaOps.Vexer.Worker/StellaOps.Vexer.Worker.csproj @@ -0,0 +1,19 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Worker/TASKS.md b/src/StellaOps.Vexer.Worker/TASKS.md index 8332c8d2..863ebf09 100644 --- a/src/StellaOps.Vexer.Worker/TASKS.md +++ b/src/StellaOps.Vexer.Worker/TASKS.md @@ -2,7 +2,7 @@ If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and # TASKS | Task | Owner(s) | Depends on | Notes | |---|---|---|---| -|VEXER-WORKER-01-001 – Worker host & scheduling|Team Vexer Worker|VEXER-STORAGE-01-003, VEXER-WEB-01-001|TODO – Scaffold Worker Service with configurable schedules per provider, honoring offline/air-gapped profiles and structured logging.| +|VEXER-WORKER-01-001 – Worker host & scheduling|Team Vexer Worker|VEXER-STORAGE-01-003, VEXER-WEB-01-001|**DONE (2025-10-17)** – Worker project bootstraps provider schedules from configuration, integrates plugin catalog discovery, and emits structured logs/metrics-ready events via `VexWorkerHostedService`; scheduling logic covered by `VexWorkerOptionsTests`.| |VEXER-WORKER-01-002 – Resume tokens & retry policy|Team Vexer Worker|VEXER-WORKER-01-001|TODO – Implement durable resume markers, exponential backoff with jitter, and quarantine for failing connectors per architecture spec.| |VEXER-WORKER-01-003 – Verification & cache GC loops|Team Vexer Worker|VEXER-WORKER-01-001, VEXER-ATTEST-01-003, VEXER-EXPORT-01-002|TODO – Add scheduled attestation re-verification and cache pruning routines, surfacing metrics for export reuse ratios.| |VEXER-WORKER-01-004 – TTL refresh & stability damper|Team Vexer Worker|VEXER-WORKER-01-001, VEXER-CORE-02-001|TODO – Monitor consensus/VEX TTLs, apply 24–48h dampers before flipping published status/score, and trigger re-resolve when base image or kernel fingerprints change.| diff --git a/src/StellaOps.sln b/src/StellaOps.sln index 65b36ae7..2bdce374 100644 --- a/src/StellaOps.sln +++ b/src/StellaOps.sln @@ -195,6 +195,48 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Export", "S EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Export.Tests", "StellaOps.Vexer.Export.Tests\StellaOps.Vexer.Export.Tests.csproj", "{06F40DA8-FEFA-4C2B-907B-155BD92BB859}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.RedHat.CSAF", "StellaOps.Vexer.Connectors.RedHat.CSAF\StellaOps.Vexer.Connectors.RedHat.CSAF.csproj", "{A2E3F03A-0CAD-4E2A-8C71-DDEBB1B7E4F7}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.RedHat.CSAF.Tests", "StellaOps.Vexer.Connectors.RedHat.CSAF.Tests\StellaOps.Vexer.Connectors.RedHat.CSAF.Tests.csproj", "{3A1AF0AD-4DAE-4D82-9CCF-2DCB83CC3679}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.Abstractions", "StellaOps.Vexer.Connectors.Abstractions\StellaOps.Vexer.Connectors.Abstractions.csproj", "{F1DF0F07-1BCB-4B55-8353-07BF8A4B2A67}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Worker", "StellaOps.Vexer.Worker\StellaOps.Vexer.Worker.csproj", "{781EC793-1DB0-4E31-95BC-12A2B373045F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Worker.Tests", "StellaOps.Vexer.Worker.Tests\StellaOps.Vexer.Worker.Tests.csproj", "{BB863E0C-50FF-41AE-9C13-4E8A1BABC62C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Formats.CSAF", "StellaOps.Vexer.Formats.CSAF\StellaOps.Vexer.Formats.CSAF.csproj", "{14E9D043-F0EF-4F68-AE83-D6F579119D9A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Formats.CSAF.Tests", "StellaOps.Vexer.Formats.CSAF.Tests\StellaOps.Vexer.Formats.CSAF.Tests.csproj", "{27E94B6E-DEF8-4B89-97CB-424703790ECE}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Formats.CycloneDX", "StellaOps.Vexer.Formats.CycloneDX\StellaOps.Vexer.Formats.CycloneDX.csproj", "{361E3E23-B215-423D-9906-A84171E20AD3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Formats.CycloneDX.Tests", "StellaOps.Vexer.Formats.CycloneDX.Tests\StellaOps.Vexer.Formats.CycloneDX.Tests.csproj", "{7A7A3480-C6C3-4A9F-AF46-1889424B9AC2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Formats.OpenVEX", "StellaOps.Vexer.Formats.OpenVEX\StellaOps.Vexer.Formats.OpenVEX.csproj", "{C3EAFCB8-0394-4B74-B9A6-3DBA4509201F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Formats.OpenVEX.Tests", "StellaOps.Vexer.Formats.OpenVEX.Tests\StellaOps.Vexer.Formats.OpenVEX.Tests.csproj", "{E86CF4A6-2463-4589-A9D8-9DF557C48367}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.Cisco.CSAF", "StellaOps.Vexer.Connectors.Cisco.CSAF\StellaOps.Vexer.Connectors.Cisco.CSAF.csproj", "{B308B94C-E01F-4449-A5A6-CD7A48E52D15}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.Cisco.CSAF.Tests", "StellaOps.Vexer.Connectors.Cisco.CSAF.Tests\StellaOps.Vexer.Connectors.Cisco.CSAF.Tests.csproj", "{9FBA3EC4-D794-48BD-82FA-0289E5A2A5FF}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.SUSE.RancherVEXHub", "StellaOps.Vexer.Connectors.SUSE.RancherVEXHub\StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.csproj", "{E076DC9C-B436-44BF-B02E-FA565086F805}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests", "StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests\StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests.csproj", "{55500025-FE82-4F97-A261-9BAEA4B10845}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.MSRC.CSAF", "StellaOps.Vexer.Connectors.MSRC.CSAF\StellaOps.Vexer.Connectors.MSRC.CSAF.csproj", "{CD12875F-9367-41BD-810C-7FBE76314F17}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.MSRC.CSAF.Tests", "StellaOps.Vexer.Connectors.MSRC.CSAF.Tests\StellaOps.Vexer.Connectors.MSRC.CSAF.Tests.csproj", "{063D3280-9918-465A-AF2D-3650A2A50D03}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.Oracle.CSAF", "StellaOps.Vexer.Connectors.Oracle.CSAF\StellaOps.Vexer.Connectors.Oracle.CSAF.csproj", "{A3EEE400-3655-4B34-915A-598E60CD55FB}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.Oracle.CSAF.Tests", "StellaOps.Vexer.Connectors.Oracle.CSAF.Tests\StellaOps.Vexer.Connectors.Oracle.CSAF.Tests.csproj", "{577025AD-2FDD-42DF-BFA2-3FC095B50539}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.Ubuntu.CSAF", "StellaOps.Vexer.Connectors.Ubuntu.CSAF\StellaOps.Vexer.Connectors.Ubuntu.CSAF.csproj", "{DD3B2076-E5E0-4533-8D27-7724225D7758}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests", "StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests\StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests.csproj", "{CADA1364-8EB1-479E-AB6F-4105C26335C8}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1333,6 +1375,258 @@ Global {06F40DA8-FEFA-4C2B-907B-155BD92BB859}.Release|x64.Build.0 = Release|Any CPU {06F40DA8-FEFA-4C2B-907B-155BD92BB859}.Release|x86.ActiveCfg = Release|Any CPU {06F40DA8-FEFA-4C2B-907B-155BD92BB859}.Release|x86.Build.0 = Release|Any CPU + {A2E3F03A-0CAD-4E2A-8C71-DDEBB1B7E4F7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A2E3F03A-0CAD-4E2A-8C71-DDEBB1B7E4F7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A2E3F03A-0CAD-4E2A-8C71-DDEBB1B7E4F7}.Debug|x64.ActiveCfg = Debug|Any CPU + {A2E3F03A-0CAD-4E2A-8C71-DDEBB1B7E4F7}.Debug|x64.Build.0 = Debug|Any CPU + {A2E3F03A-0CAD-4E2A-8C71-DDEBB1B7E4F7}.Debug|x86.ActiveCfg = Debug|Any CPU + {A2E3F03A-0CAD-4E2A-8C71-DDEBB1B7E4F7}.Debug|x86.Build.0 = Debug|Any CPU + {A2E3F03A-0CAD-4E2A-8C71-DDEBB1B7E4F7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A2E3F03A-0CAD-4E2A-8C71-DDEBB1B7E4F7}.Release|Any CPU.Build.0 = Release|Any CPU + {A2E3F03A-0CAD-4E2A-8C71-DDEBB1B7E4F7}.Release|x64.ActiveCfg = Release|Any CPU + {A2E3F03A-0CAD-4E2A-8C71-DDEBB1B7E4F7}.Release|x64.Build.0 = Release|Any CPU + {A2E3F03A-0CAD-4E2A-8C71-DDEBB1B7E4F7}.Release|x86.ActiveCfg = Release|Any CPU + {A2E3F03A-0CAD-4E2A-8C71-DDEBB1B7E4F7}.Release|x86.Build.0 = Release|Any CPU + {3A1AF0AD-4DAE-4D82-9CCF-2DCB83CC3679}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3A1AF0AD-4DAE-4D82-9CCF-2DCB83CC3679}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3A1AF0AD-4DAE-4D82-9CCF-2DCB83CC3679}.Debug|x64.ActiveCfg = Debug|Any CPU + {3A1AF0AD-4DAE-4D82-9CCF-2DCB83CC3679}.Debug|x64.Build.0 = Debug|Any CPU + {3A1AF0AD-4DAE-4D82-9CCF-2DCB83CC3679}.Debug|x86.ActiveCfg = Debug|Any CPU + {3A1AF0AD-4DAE-4D82-9CCF-2DCB83CC3679}.Debug|x86.Build.0 = Debug|Any CPU + {3A1AF0AD-4DAE-4D82-9CCF-2DCB83CC3679}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3A1AF0AD-4DAE-4D82-9CCF-2DCB83CC3679}.Release|Any CPU.Build.0 = Release|Any CPU + {3A1AF0AD-4DAE-4D82-9CCF-2DCB83CC3679}.Release|x64.ActiveCfg = Release|Any CPU + {3A1AF0AD-4DAE-4D82-9CCF-2DCB83CC3679}.Release|x64.Build.0 = Release|Any CPU + {3A1AF0AD-4DAE-4D82-9CCF-2DCB83CC3679}.Release|x86.ActiveCfg = Release|Any CPU + {3A1AF0AD-4DAE-4D82-9CCF-2DCB83CC3679}.Release|x86.Build.0 = Release|Any CPU + {F1DF0F07-1BCB-4B55-8353-07BF8A4B2A67}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F1DF0F07-1BCB-4B55-8353-07BF8A4B2A67}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F1DF0F07-1BCB-4B55-8353-07BF8A4B2A67}.Debug|x64.ActiveCfg = Debug|Any CPU + {F1DF0F07-1BCB-4B55-8353-07BF8A4B2A67}.Debug|x64.Build.0 = Debug|Any CPU + {F1DF0F07-1BCB-4B55-8353-07BF8A4B2A67}.Debug|x86.ActiveCfg = Debug|Any CPU + {F1DF0F07-1BCB-4B55-8353-07BF8A4B2A67}.Debug|x86.Build.0 = Debug|Any CPU + {F1DF0F07-1BCB-4B55-8353-07BF8A4B2A67}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F1DF0F07-1BCB-4B55-8353-07BF8A4B2A67}.Release|Any CPU.Build.0 = Release|Any CPU + {F1DF0F07-1BCB-4B55-8353-07BF8A4B2A67}.Release|x64.ActiveCfg = Release|Any CPU + {F1DF0F07-1BCB-4B55-8353-07BF8A4B2A67}.Release|x64.Build.0 = Release|Any CPU + {F1DF0F07-1BCB-4B55-8353-07BF8A4B2A67}.Release|x86.ActiveCfg = Release|Any CPU + {F1DF0F07-1BCB-4B55-8353-07BF8A4B2A67}.Release|x86.Build.0 = Release|Any CPU + {781EC793-1DB0-4E31-95BC-12A2B373045F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {781EC793-1DB0-4E31-95BC-12A2B373045F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {781EC793-1DB0-4E31-95BC-12A2B373045F}.Debug|x64.ActiveCfg = Debug|Any CPU + {781EC793-1DB0-4E31-95BC-12A2B373045F}.Debug|x64.Build.0 = Debug|Any CPU + {781EC793-1DB0-4E31-95BC-12A2B373045F}.Debug|x86.ActiveCfg = Debug|Any CPU + {781EC793-1DB0-4E31-95BC-12A2B373045F}.Debug|x86.Build.0 = Debug|Any CPU + {781EC793-1DB0-4E31-95BC-12A2B373045F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {781EC793-1DB0-4E31-95BC-12A2B373045F}.Release|Any CPU.Build.0 = Release|Any CPU + {781EC793-1DB0-4E31-95BC-12A2B373045F}.Release|x64.ActiveCfg = Release|Any CPU + {781EC793-1DB0-4E31-95BC-12A2B373045F}.Release|x64.Build.0 = Release|Any CPU + {781EC793-1DB0-4E31-95BC-12A2B373045F}.Release|x86.ActiveCfg = Release|Any CPU + {781EC793-1DB0-4E31-95BC-12A2B373045F}.Release|x86.Build.0 = Release|Any CPU + {BB863E0C-50FF-41AE-9C13-4E8A1BABC62C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BB863E0C-50FF-41AE-9C13-4E8A1BABC62C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BB863E0C-50FF-41AE-9C13-4E8A1BABC62C}.Debug|x64.ActiveCfg = Debug|Any CPU + {BB863E0C-50FF-41AE-9C13-4E8A1BABC62C}.Debug|x64.Build.0 = Debug|Any CPU + {BB863E0C-50FF-41AE-9C13-4E8A1BABC62C}.Debug|x86.ActiveCfg = Debug|Any CPU + {BB863E0C-50FF-41AE-9C13-4E8A1BABC62C}.Debug|x86.Build.0 = Debug|Any CPU + {BB863E0C-50FF-41AE-9C13-4E8A1BABC62C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BB863E0C-50FF-41AE-9C13-4E8A1BABC62C}.Release|Any CPU.Build.0 = Release|Any CPU + {BB863E0C-50FF-41AE-9C13-4E8A1BABC62C}.Release|x64.ActiveCfg = Release|Any CPU + {BB863E0C-50FF-41AE-9C13-4E8A1BABC62C}.Release|x64.Build.0 = Release|Any CPU + {BB863E0C-50FF-41AE-9C13-4E8A1BABC62C}.Release|x86.ActiveCfg = Release|Any CPU + {BB863E0C-50FF-41AE-9C13-4E8A1BABC62C}.Release|x86.Build.0 = Release|Any CPU + {14E9D043-F0EF-4F68-AE83-D6F579119D9A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {14E9D043-F0EF-4F68-AE83-D6F579119D9A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {14E9D043-F0EF-4F68-AE83-D6F579119D9A}.Debug|x64.ActiveCfg = Debug|Any CPU + {14E9D043-F0EF-4F68-AE83-D6F579119D9A}.Debug|x64.Build.0 = Debug|Any CPU + {14E9D043-F0EF-4F68-AE83-D6F579119D9A}.Debug|x86.ActiveCfg = Debug|Any CPU + {14E9D043-F0EF-4F68-AE83-D6F579119D9A}.Debug|x86.Build.0 = Debug|Any CPU + {14E9D043-F0EF-4F68-AE83-D6F579119D9A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {14E9D043-F0EF-4F68-AE83-D6F579119D9A}.Release|Any CPU.Build.0 = Release|Any CPU + {14E9D043-F0EF-4F68-AE83-D6F579119D9A}.Release|x64.ActiveCfg = Release|Any CPU + {14E9D043-F0EF-4F68-AE83-D6F579119D9A}.Release|x64.Build.0 = Release|Any CPU + {14E9D043-F0EF-4F68-AE83-D6F579119D9A}.Release|x86.ActiveCfg = Release|Any CPU + {14E9D043-F0EF-4F68-AE83-D6F579119D9A}.Release|x86.Build.0 = Release|Any CPU + {27E94B6E-DEF8-4B89-97CB-424703790ECE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {27E94B6E-DEF8-4B89-97CB-424703790ECE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {27E94B6E-DEF8-4B89-97CB-424703790ECE}.Debug|x64.ActiveCfg = Debug|Any CPU + {27E94B6E-DEF8-4B89-97CB-424703790ECE}.Debug|x64.Build.0 = Debug|Any CPU + {27E94B6E-DEF8-4B89-97CB-424703790ECE}.Debug|x86.ActiveCfg = Debug|Any CPU + {27E94B6E-DEF8-4B89-97CB-424703790ECE}.Debug|x86.Build.0 = Debug|Any CPU + {27E94B6E-DEF8-4B89-97CB-424703790ECE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {27E94B6E-DEF8-4B89-97CB-424703790ECE}.Release|Any CPU.Build.0 = Release|Any CPU + {27E94B6E-DEF8-4B89-97CB-424703790ECE}.Release|x64.ActiveCfg = Release|Any CPU + {27E94B6E-DEF8-4B89-97CB-424703790ECE}.Release|x64.Build.0 = Release|Any CPU + {27E94B6E-DEF8-4B89-97CB-424703790ECE}.Release|x86.ActiveCfg = Release|Any CPU + {27E94B6E-DEF8-4B89-97CB-424703790ECE}.Release|x86.Build.0 = Release|Any CPU + {361E3E23-B215-423D-9906-A84171E20AD3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {361E3E23-B215-423D-9906-A84171E20AD3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {361E3E23-B215-423D-9906-A84171E20AD3}.Debug|x64.ActiveCfg = Debug|Any CPU + {361E3E23-B215-423D-9906-A84171E20AD3}.Debug|x64.Build.0 = Debug|Any CPU + {361E3E23-B215-423D-9906-A84171E20AD3}.Debug|x86.ActiveCfg = Debug|Any CPU + {361E3E23-B215-423D-9906-A84171E20AD3}.Debug|x86.Build.0 = Debug|Any CPU + {361E3E23-B215-423D-9906-A84171E20AD3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {361E3E23-B215-423D-9906-A84171E20AD3}.Release|Any CPU.Build.0 = Release|Any CPU + {361E3E23-B215-423D-9906-A84171E20AD3}.Release|x64.ActiveCfg = Release|Any CPU + {361E3E23-B215-423D-9906-A84171E20AD3}.Release|x64.Build.0 = Release|Any CPU + {361E3E23-B215-423D-9906-A84171E20AD3}.Release|x86.ActiveCfg = Release|Any CPU + {361E3E23-B215-423D-9906-A84171E20AD3}.Release|x86.Build.0 = Release|Any CPU + {7A7A3480-C6C3-4A9F-AF46-1889424B9AC2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7A7A3480-C6C3-4A9F-AF46-1889424B9AC2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7A7A3480-C6C3-4A9F-AF46-1889424B9AC2}.Debug|x64.ActiveCfg = Debug|Any CPU + {7A7A3480-C6C3-4A9F-AF46-1889424B9AC2}.Debug|x64.Build.0 = Debug|Any CPU + {7A7A3480-C6C3-4A9F-AF46-1889424B9AC2}.Debug|x86.ActiveCfg = Debug|Any CPU + {7A7A3480-C6C3-4A9F-AF46-1889424B9AC2}.Debug|x86.Build.0 = Debug|Any CPU + {7A7A3480-C6C3-4A9F-AF46-1889424B9AC2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7A7A3480-C6C3-4A9F-AF46-1889424B9AC2}.Release|Any CPU.Build.0 = Release|Any CPU + {7A7A3480-C6C3-4A9F-AF46-1889424B9AC2}.Release|x64.ActiveCfg = Release|Any CPU + {7A7A3480-C6C3-4A9F-AF46-1889424B9AC2}.Release|x64.Build.0 = Release|Any CPU + {7A7A3480-C6C3-4A9F-AF46-1889424B9AC2}.Release|x86.ActiveCfg = Release|Any CPU + {7A7A3480-C6C3-4A9F-AF46-1889424B9AC2}.Release|x86.Build.0 = Release|Any CPU + {C3EAFCB8-0394-4B74-B9A6-3DBA4509201F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C3EAFCB8-0394-4B74-B9A6-3DBA4509201F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C3EAFCB8-0394-4B74-B9A6-3DBA4509201F}.Debug|x64.ActiveCfg = Debug|Any CPU + {C3EAFCB8-0394-4B74-B9A6-3DBA4509201F}.Debug|x64.Build.0 = Debug|Any CPU + {C3EAFCB8-0394-4B74-B9A6-3DBA4509201F}.Debug|x86.ActiveCfg = Debug|Any CPU + {C3EAFCB8-0394-4B74-B9A6-3DBA4509201F}.Debug|x86.Build.0 = Debug|Any CPU + {C3EAFCB8-0394-4B74-B9A6-3DBA4509201F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C3EAFCB8-0394-4B74-B9A6-3DBA4509201F}.Release|Any CPU.Build.0 = Release|Any CPU + {C3EAFCB8-0394-4B74-B9A6-3DBA4509201F}.Release|x64.ActiveCfg = Release|Any CPU + {C3EAFCB8-0394-4B74-B9A6-3DBA4509201F}.Release|x64.Build.0 = Release|Any CPU + {C3EAFCB8-0394-4B74-B9A6-3DBA4509201F}.Release|x86.ActiveCfg = Release|Any CPU + {C3EAFCB8-0394-4B74-B9A6-3DBA4509201F}.Release|x86.Build.0 = Release|Any CPU + {E86CF4A6-2463-4589-A9D8-9DF557C48367}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E86CF4A6-2463-4589-A9D8-9DF557C48367}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E86CF4A6-2463-4589-A9D8-9DF557C48367}.Debug|x64.ActiveCfg = Debug|Any CPU + {E86CF4A6-2463-4589-A9D8-9DF557C48367}.Debug|x64.Build.0 = Debug|Any CPU + {E86CF4A6-2463-4589-A9D8-9DF557C48367}.Debug|x86.ActiveCfg = Debug|Any CPU + {E86CF4A6-2463-4589-A9D8-9DF557C48367}.Debug|x86.Build.0 = Debug|Any CPU + {E86CF4A6-2463-4589-A9D8-9DF557C48367}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E86CF4A6-2463-4589-A9D8-9DF557C48367}.Release|Any CPU.Build.0 = Release|Any CPU + {E86CF4A6-2463-4589-A9D8-9DF557C48367}.Release|x64.ActiveCfg = Release|Any CPU + {E86CF4A6-2463-4589-A9D8-9DF557C48367}.Release|x64.Build.0 = Release|Any CPU + {E86CF4A6-2463-4589-A9D8-9DF557C48367}.Release|x86.ActiveCfg = Release|Any CPU + {E86CF4A6-2463-4589-A9D8-9DF557C48367}.Release|x86.Build.0 = Release|Any CPU + {B308B94C-E01F-4449-A5A6-CD7A48E52D15}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B308B94C-E01F-4449-A5A6-CD7A48E52D15}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B308B94C-E01F-4449-A5A6-CD7A48E52D15}.Debug|x64.ActiveCfg = Debug|Any CPU + {B308B94C-E01F-4449-A5A6-CD7A48E52D15}.Debug|x64.Build.0 = Debug|Any CPU + {B308B94C-E01F-4449-A5A6-CD7A48E52D15}.Debug|x86.ActiveCfg = Debug|Any CPU + {B308B94C-E01F-4449-A5A6-CD7A48E52D15}.Debug|x86.Build.0 = Debug|Any CPU + {B308B94C-E01F-4449-A5A6-CD7A48E52D15}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B308B94C-E01F-4449-A5A6-CD7A48E52D15}.Release|Any CPU.Build.0 = Release|Any CPU + {B308B94C-E01F-4449-A5A6-CD7A48E52D15}.Release|x64.ActiveCfg = Release|Any CPU + {B308B94C-E01F-4449-A5A6-CD7A48E52D15}.Release|x64.Build.0 = Release|Any CPU + {B308B94C-E01F-4449-A5A6-CD7A48E52D15}.Release|x86.ActiveCfg = Release|Any CPU + {B308B94C-E01F-4449-A5A6-CD7A48E52D15}.Release|x86.Build.0 = Release|Any CPU + {9FBA3EC4-D794-48BD-82FA-0289E5A2A5FF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9FBA3EC4-D794-48BD-82FA-0289E5A2A5FF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9FBA3EC4-D794-48BD-82FA-0289E5A2A5FF}.Debug|x64.ActiveCfg = Debug|Any CPU + {9FBA3EC4-D794-48BD-82FA-0289E5A2A5FF}.Debug|x64.Build.0 = Debug|Any CPU + {9FBA3EC4-D794-48BD-82FA-0289E5A2A5FF}.Debug|x86.ActiveCfg = Debug|Any CPU + {9FBA3EC4-D794-48BD-82FA-0289E5A2A5FF}.Debug|x86.Build.0 = Debug|Any CPU + {9FBA3EC4-D794-48BD-82FA-0289E5A2A5FF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9FBA3EC4-D794-48BD-82FA-0289E5A2A5FF}.Release|Any CPU.Build.0 = Release|Any CPU + {9FBA3EC4-D794-48BD-82FA-0289E5A2A5FF}.Release|x64.ActiveCfg = Release|Any CPU + {9FBA3EC4-D794-48BD-82FA-0289E5A2A5FF}.Release|x64.Build.0 = Release|Any CPU + {9FBA3EC4-D794-48BD-82FA-0289E5A2A5FF}.Release|x86.ActiveCfg = Release|Any CPU + {9FBA3EC4-D794-48BD-82FA-0289E5A2A5FF}.Release|x86.Build.0 = Release|Any CPU + {E076DC9C-B436-44BF-B02E-FA565086F805}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E076DC9C-B436-44BF-B02E-FA565086F805}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E076DC9C-B436-44BF-B02E-FA565086F805}.Debug|x64.ActiveCfg = Debug|Any CPU + {E076DC9C-B436-44BF-B02E-FA565086F805}.Debug|x64.Build.0 = Debug|Any CPU + {E076DC9C-B436-44BF-B02E-FA565086F805}.Debug|x86.ActiveCfg = Debug|Any CPU + {E076DC9C-B436-44BF-B02E-FA565086F805}.Debug|x86.Build.0 = Debug|Any CPU + {E076DC9C-B436-44BF-B02E-FA565086F805}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E076DC9C-B436-44BF-B02E-FA565086F805}.Release|Any CPU.Build.0 = Release|Any CPU + {E076DC9C-B436-44BF-B02E-FA565086F805}.Release|x64.ActiveCfg = Release|Any CPU + {E076DC9C-B436-44BF-B02E-FA565086F805}.Release|x64.Build.0 = Release|Any CPU + {E076DC9C-B436-44BF-B02E-FA565086F805}.Release|x86.ActiveCfg = Release|Any CPU + {E076DC9C-B436-44BF-B02E-FA565086F805}.Release|x86.Build.0 = Release|Any CPU + {55500025-FE82-4F97-A261-9BAEA4B10845}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {55500025-FE82-4F97-A261-9BAEA4B10845}.Debug|Any CPU.Build.0 = Debug|Any CPU + {55500025-FE82-4F97-A261-9BAEA4B10845}.Debug|x64.ActiveCfg = Debug|Any CPU + {55500025-FE82-4F97-A261-9BAEA4B10845}.Debug|x64.Build.0 = Debug|Any CPU + {55500025-FE82-4F97-A261-9BAEA4B10845}.Debug|x86.ActiveCfg = Debug|Any CPU + {55500025-FE82-4F97-A261-9BAEA4B10845}.Debug|x86.Build.0 = Debug|Any CPU + {55500025-FE82-4F97-A261-9BAEA4B10845}.Release|Any CPU.ActiveCfg = Release|Any CPU + {55500025-FE82-4F97-A261-9BAEA4B10845}.Release|Any CPU.Build.0 = Release|Any CPU + {55500025-FE82-4F97-A261-9BAEA4B10845}.Release|x64.ActiveCfg = Release|Any CPU + {55500025-FE82-4F97-A261-9BAEA4B10845}.Release|x64.Build.0 = Release|Any CPU + {55500025-FE82-4F97-A261-9BAEA4B10845}.Release|x86.ActiveCfg = Release|Any CPU + {55500025-FE82-4F97-A261-9BAEA4B10845}.Release|x86.Build.0 = Release|Any CPU + {CD12875F-9367-41BD-810C-7FBE76314F17}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CD12875F-9367-41BD-810C-7FBE76314F17}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CD12875F-9367-41BD-810C-7FBE76314F17}.Debug|x64.ActiveCfg = Debug|Any CPU + {CD12875F-9367-41BD-810C-7FBE76314F17}.Debug|x64.Build.0 = Debug|Any CPU + {CD12875F-9367-41BD-810C-7FBE76314F17}.Debug|x86.ActiveCfg = Debug|Any CPU + {CD12875F-9367-41BD-810C-7FBE76314F17}.Debug|x86.Build.0 = Debug|Any CPU + {CD12875F-9367-41BD-810C-7FBE76314F17}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CD12875F-9367-41BD-810C-7FBE76314F17}.Release|Any CPU.Build.0 = Release|Any CPU + {CD12875F-9367-41BD-810C-7FBE76314F17}.Release|x64.ActiveCfg = Release|Any CPU + {CD12875F-9367-41BD-810C-7FBE76314F17}.Release|x64.Build.0 = Release|Any CPU + {CD12875F-9367-41BD-810C-7FBE76314F17}.Release|x86.ActiveCfg = Release|Any CPU + {CD12875F-9367-41BD-810C-7FBE76314F17}.Release|x86.Build.0 = Release|Any CPU + {063D3280-9918-465A-AF2D-3650A2A50D03}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {063D3280-9918-465A-AF2D-3650A2A50D03}.Debug|Any CPU.Build.0 = Debug|Any CPU + {063D3280-9918-465A-AF2D-3650A2A50D03}.Debug|x64.ActiveCfg = Debug|Any CPU + {063D3280-9918-465A-AF2D-3650A2A50D03}.Debug|x64.Build.0 = Debug|Any CPU + {063D3280-9918-465A-AF2D-3650A2A50D03}.Debug|x86.ActiveCfg = Debug|Any CPU + {063D3280-9918-465A-AF2D-3650A2A50D03}.Debug|x86.Build.0 = Debug|Any CPU + {063D3280-9918-465A-AF2D-3650A2A50D03}.Release|Any CPU.ActiveCfg = Release|Any CPU + {063D3280-9918-465A-AF2D-3650A2A50D03}.Release|Any CPU.Build.0 = Release|Any CPU + {063D3280-9918-465A-AF2D-3650A2A50D03}.Release|x64.ActiveCfg = Release|Any CPU + {063D3280-9918-465A-AF2D-3650A2A50D03}.Release|x64.Build.0 = Release|Any CPU + {063D3280-9918-465A-AF2D-3650A2A50D03}.Release|x86.ActiveCfg = Release|Any CPU + {063D3280-9918-465A-AF2D-3650A2A50D03}.Release|x86.Build.0 = Release|Any CPU + {A3EEE400-3655-4B34-915A-598E60CD55FB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A3EEE400-3655-4B34-915A-598E60CD55FB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A3EEE400-3655-4B34-915A-598E60CD55FB}.Debug|x64.ActiveCfg = Debug|Any CPU + {A3EEE400-3655-4B34-915A-598E60CD55FB}.Debug|x64.Build.0 = Debug|Any CPU + {A3EEE400-3655-4B34-915A-598E60CD55FB}.Debug|x86.ActiveCfg = Debug|Any CPU + {A3EEE400-3655-4B34-915A-598E60CD55FB}.Debug|x86.Build.0 = Debug|Any CPU + {A3EEE400-3655-4B34-915A-598E60CD55FB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A3EEE400-3655-4B34-915A-598E60CD55FB}.Release|Any CPU.Build.0 = Release|Any CPU + {A3EEE400-3655-4B34-915A-598E60CD55FB}.Release|x64.ActiveCfg = Release|Any CPU + {A3EEE400-3655-4B34-915A-598E60CD55FB}.Release|x64.Build.0 = Release|Any CPU + {A3EEE400-3655-4B34-915A-598E60CD55FB}.Release|x86.ActiveCfg = Release|Any CPU + {A3EEE400-3655-4B34-915A-598E60CD55FB}.Release|x86.Build.0 = Release|Any CPU + {577025AD-2FDD-42DF-BFA2-3FC095B50539}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {577025AD-2FDD-42DF-BFA2-3FC095B50539}.Debug|Any CPU.Build.0 = Debug|Any CPU + {577025AD-2FDD-42DF-BFA2-3FC095B50539}.Debug|x64.ActiveCfg = Debug|Any CPU + {577025AD-2FDD-42DF-BFA2-3FC095B50539}.Debug|x64.Build.0 = Debug|Any CPU + {577025AD-2FDD-42DF-BFA2-3FC095B50539}.Debug|x86.ActiveCfg = Debug|Any CPU + {577025AD-2FDD-42DF-BFA2-3FC095B50539}.Debug|x86.Build.0 = Debug|Any CPU + {577025AD-2FDD-42DF-BFA2-3FC095B50539}.Release|Any CPU.ActiveCfg = Release|Any CPU + {577025AD-2FDD-42DF-BFA2-3FC095B50539}.Release|Any CPU.Build.0 = Release|Any CPU + {577025AD-2FDD-42DF-BFA2-3FC095B50539}.Release|x64.ActiveCfg = Release|Any CPU + {577025AD-2FDD-42DF-BFA2-3FC095B50539}.Release|x64.Build.0 = Release|Any CPU + {577025AD-2FDD-42DF-BFA2-3FC095B50539}.Release|x86.ActiveCfg = Release|Any CPU + {577025AD-2FDD-42DF-BFA2-3FC095B50539}.Release|x86.Build.0 = Release|Any CPU + {DD3B2076-E5E0-4533-8D27-7724225D7758}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DD3B2076-E5E0-4533-8D27-7724225D7758}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DD3B2076-E5E0-4533-8D27-7724225D7758}.Debug|x64.ActiveCfg = Debug|Any CPU + {DD3B2076-E5E0-4533-8D27-7724225D7758}.Debug|x64.Build.0 = Debug|Any CPU + {DD3B2076-E5E0-4533-8D27-7724225D7758}.Debug|x86.ActiveCfg = Debug|Any CPU + {DD3B2076-E5E0-4533-8D27-7724225D7758}.Debug|x86.Build.0 = Debug|Any CPU + {DD3B2076-E5E0-4533-8D27-7724225D7758}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DD3B2076-E5E0-4533-8D27-7724225D7758}.Release|Any CPU.Build.0 = Release|Any CPU + {DD3B2076-E5E0-4533-8D27-7724225D7758}.Release|x64.ActiveCfg = Release|Any CPU + {DD3B2076-E5E0-4533-8D27-7724225D7758}.Release|x64.Build.0 = Release|Any CPU + {DD3B2076-E5E0-4533-8D27-7724225D7758}.Release|x86.ActiveCfg = Release|Any CPU + {DD3B2076-E5E0-4533-8D27-7724225D7758}.Release|x86.Build.0 = Release|Any CPU + {CADA1364-8EB1-479E-AB6F-4105C26335C8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CADA1364-8EB1-479E-AB6F-4105C26335C8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CADA1364-8EB1-479E-AB6F-4105C26335C8}.Debug|x64.ActiveCfg = Debug|Any CPU + {CADA1364-8EB1-479E-AB6F-4105C26335C8}.Debug|x64.Build.0 = Debug|Any CPU + {CADA1364-8EB1-479E-AB6F-4105C26335C8}.Debug|x86.ActiveCfg = Debug|Any CPU + {CADA1364-8EB1-479E-AB6F-4105C26335C8}.Debug|x86.Build.0 = Debug|Any CPU + {CADA1364-8EB1-479E-AB6F-4105C26335C8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CADA1364-8EB1-479E-AB6F-4105C26335C8}.Release|Any CPU.Build.0 = Release|Any CPU + {CADA1364-8EB1-479E-AB6F-4105C26335C8}.Release|x64.ActiveCfg = Release|Any CPU + {CADA1364-8EB1-479E-AB6F-4105C26335C8}.Release|x64.Build.0 = Release|Any CPU + {CADA1364-8EB1-479E-AB6F-4105C26335C8}.Release|x86.ActiveCfg = Release|Any CPU + {CADA1364-8EB1-479E-AB6F-4105C26335C8}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1425,5 +1719,8 @@ Global {FAB78D21-7372-48FE-B2C3-DE1807F1157D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {EADFA337-B0FA-4712-A24A-7C08235BDF98} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {110F7EC2-3149-4D1B-A972-E69E79F1EBF5} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {A2E3F03A-0CAD-4E2A-8C71-DDEBB1B7E4F7} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {3A1AF0AD-4DAE-4D82-9CCF-2DCB83CC3679} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {F1DF0F07-1BCB-4B55-8353-07BF8A4B2A67} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} EndGlobalSection EndGlobal diff --git a/tools/certbund_offline_snapshot.py b/tools/certbund_offline_snapshot.py new file mode 100644 index 00000000..8d7aafc4 --- /dev/null +++ b/tools/certbund_offline_snapshot.py @@ -0,0 +1,444 @@ +#!/usr/bin/env python3 +""" +Capture CERT-Bund search/export JSON snapshots and generate Offline Kit manifests. + +The script can bootstrap a session against https://wid.cert-bund.de, fetch +paginated search results plus per-year export payloads, and emit a manifest +that records source, date range, SHA-256, and capture timestamps for each artefact. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from http.cookiejar import MozillaCookieJar +from typing import Any, Dict, Iterable, List, Optional + + +PORTAL_ROOT = "https://wid.cert-bund.de/portal/" +SEARCH_ENDPOINT = "https://wid.cert-bund.de/portal/api/securityadvisory/search" +EXPORT_ENDPOINT = "https://wid.cert-bund.de/portal/api/securityadvisory/export" +CSRF_ENDPOINT = "https://wid.cert-bund.de/portal/api/security/csrf" +USER_AGENT = "StellaOps.CertBundOffline/0.1" + +UTC = dt.timezone.utc + + +class CertBundClient: + def __init__( + self, + cookie_file: Optional[Path] = None, + xsrf_token: Optional[str] = None, + auto_bootstrap: bool = True, + ) -> None: + self.cookie_path = cookie_file + self.cookie_jar = MozillaCookieJar() + + if self.cookie_path and self.cookie_path.exists(): + self.cookie_jar.load(self.cookie_path, ignore_discard=True, ignore_expires=True) + + self.opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(self.cookie_jar)) + self.opener.addheaders = [("User-Agent", USER_AGENT)] + + self._xsrf_token = xsrf_token + self.auto_bootstrap = auto_bootstrap + + if self.auto_bootstrap and not self._xsrf_token: + self._bootstrap() + + @property + def xsrf_token(self) -> str: + if self._xsrf_token: + return self._xsrf_token + + token = _extract_cookie_value(self.cookie_jar, "XSRF-TOKEN") + if token: + self._xsrf_token = token + return token + + raise RuntimeError( + "CERT-Bund XSRF token not available. Provide --xsrf-token or a cookie file " + "containing XSRF-TOKEN (see docs/ops/feedser-certbund-operations.md)." + ) + + def fetch_search_pages( + self, + destination: Path, + page_size: int, + max_pages: int, + ) -> None: + destination.mkdir(parents=True, exist_ok=True) + + for page in range(max_pages): + payload = { + "page": page, + "size": page_size, + "sort": ["published,desc"], + } + try: + document = self._post_json(SEARCH_ENDPOINT, payload) + except urllib.error.HTTPError as exc: + raise RuntimeError( + f"Failed to fetch CERT-Bund search page {page}: HTTP {exc.code}. " + "Double-check the XSRF token or portal cookies." + ) from exc + + content = document.get("content") or [] + if not content and page > 0: + break + + file_path = destination / f"certbund-search-page-{page:02d}.json" + _write_pretty_json(file_path, document) + print(f"[certbund] wrote search page {page:02d} → {file_path}") + + if not content: + break + + self._persist_cookies() + + def fetch_exports(self, destination: Path, start_year: int, end_year: int) -> None: + destination.mkdir(parents=True, exist_ok=True) + + for year in range(start_year, end_year + 1): + from_value = f"{year}-01-01" + to_value = f"{year}-12-31" + query = urllib.parse.urlencode({"format": "json", "from": from_value, "to": to_value}) + url = f"{EXPORT_ENDPOINT}?{query}" + try: + document = self._get_json(url) + except urllib.error.HTTPError as exc: + raise RuntimeError( + f"Failed to fetch CERT-Bund export for {year}: HTTP {exc.code}. " + "Ensure the XSRF token and cookies are valid." + ) from exc + + file_path = destination / f"certbund-export-{year}.json" + _write_pretty_json(file_path, document) + print(f"[certbund] wrote export {year} → {file_path}") + + self._persist_cookies() + + def _bootstrap(self) -> None: + try: + self._request("GET", PORTAL_ROOT, headers={"Accept": "text/html,application/xhtml+xml"}) + except urllib.error.HTTPError as exc: + raise RuntimeError(f"Failed to bootstrap CERT-Bund session: HTTP {exc.code}") from exc + + # First attempt to obtain CSRF token directly. + self._attempt_csrf_fetch() + + if _extract_cookie_value(self.cookie_jar, "XSRF-TOKEN"): + return + + # If the token is still missing, trigger the search endpoint once (likely 403) + # to make the portal materialise JSESSIONID, then retry token acquisition. + try: + payload = {"page": 0, "size": 1, "sort": ["published,desc"]} + self._post_json(SEARCH_ENDPOINT, payload, include_token=False) + except urllib.error.HTTPError: + pass + + self._attempt_csrf_fetch() + + token = _extract_cookie_value(self.cookie_jar, "XSRF-TOKEN") + if token: + self._xsrf_token = token + else: + print( + "[certbund] warning: automatic XSRF token retrieval failed. " + "Supply --xsrf-token or reuse a browser-exported cookies file.", + file=sys.stderr, + ) + + def _attempt_csrf_fetch(self) -> None: + headers = { + "Accept": "application/json, text/plain, */*", + "X-Requested-With": "XMLHttpRequest", + "Origin": "https://wid.cert-bund.de", + "Referer": PORTAL_ROOT, + } + try: + self._request("GET", CSRF_ENDPOINT, headers=headers) + except urllib.error.HTTPError: + pass + + def _request(self, method: str, url: str, data: Optional[bytes] = None, headers: Optional[Dict[str, str]] = None) -> bytes: + request = urllib.request.Request(url, data=data, method=method) + default_headers = { + "User-Agent": USER_AGENT, + "Accept": "application/json", + } + for key, value in default_headers.items(): + request.add_header(key, value) + + if headers: + for key, value in headers.items(): + request.add_header(key, value) + + return self.opener.open(request, timeout=60).read() + + def _post_json(self, url: str, payload: Dict[str, Any], include_token: bool = True) -> Dict[str, Any]: + data = json.dumps(payload).encode("utf-8") + headers = { + "Content-Type": "application/json", + "Accept": "application/json", + "X-Requested-With": "XMLHttpRequest", + "Origin": "https://wid.cert-bund.de", + "Referer": PORTAL_ROOT, + } + if include_token: + headers["X-XSRF-TOKEN"] = self.xsrf_token + + raw = self._request("POST", url, data=data, headers=headers) + return json.loads(raw.decode("utf-8")) + + def _get_json(self, url: str) -> Any: + headers = { + "Accept": "application/json", + "X-Requested-With": "XMLHttpRequest", + "Referer": PORTAL_ROOT, + } + headers["X-XSRF-TOKEN"] = self.xsrf_token + + raw = self._request("GET", url, headers=headers) + return json.loads(raw.decode("utf-8")) + + def _persist_cookies(self) -> None: + if not self.cookie_path: + return + + self.cookie_path.parent.mkdir(parents=True, exist_ok=True) + self.cookie_jar.save(self.cookie_path, ignore_discard=True, ignore_expires=True) + + +def _extract_cookie_value(jar: MozillaCookieJar, name: str) -> Optional[str]: + for cookie in jar: + if cookie.name == name: + return cookie.value + return None + + +def _write_pretty_json(path: Path, document: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + json.dump(document, handle, ensure_ascii=False, indent=2, sort_keys=True) + handle.write("\n") + + +def scan_artifacts(root: Path) -> List[Dict[str, Any]]: + records: List[Dict[str, Any]] = [] + search_dir = root / "search" + export_dir = root / "export" + + if search_dir.exists(): + for file_path in sorted(search_dir.glob("certbund-search-page-*.json")): + record = _build_search_record(file_path) + records.append(record) + + if export_dir.exists(): + for file_path in sorted(export_dir.glob("certbund-export-*.json")): + record = _build_export_record(file_path) + records.append(record) + + return records + + +def _build_search_record(path: Path) -> Dict[str, Any]: + with path.open("r", encoding="utf-8") as handle: + data = json.load(handle) + + content = data.get("content") or [] + published_values: List[str] = [] + for item in content: + published = ( + item.get("published") + or item.get("publishedAt") + or item.get("datePublished") + or item.get("published_date") + ) + if isinstance(published, str): + published_values.append(published) + + if published_values: + try: + ordered = sorted(_parse_iso_timestamp(value) for value in published_values if value) + range_from = ordered[0].isoformat() + range_to = ordered[-1].isoformat() + except ValueError: + range_from = range_to = None + else: + range_from = range_to = None + + return { + "type": "search", + "path": path, + "source": "feedser.cert-bund.search", + "itemCount": len(content), + "from": range_from, + "to": range_to, + "capturedAt": _timestamp_from_stat(path), + } + + +def _build_export_record(path: Path) -> Dict[str, Any]: + year = _extract_year_from_filename(path.name) + if year is not None: + from_value = f"{year}-01-01" + to_value = f"{year}-12-31" + else: + from_value = None + to_value = None + + return { + "type": "export", + "path": path, + "source": "feedser.cert-bund.export", + "itemCount": None, + "from": from_value, + "to": to_value, + "capturedAt": _timestamp_from_stat(path), + } + + +def _timestamp_from_stat(path: Path) -> str: + stat = path.stat() + return dt.datetime.fromtimestamp(stat.st_mtime, tz=UTC).isoformat() + + +def _extract_year_from_filename(name: str) -> Optional[int]: + stem = Path(name).stem + parts = stem.split("-") + if parts and parts[-1].isdigit() and len(parts[-1]) == 4: + return int(parts[-1]) + return None + + +def _parse_iso_timestamp(value: str) -> dt.datetime: + try: + return dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + # Fallback for formats like 2025-10-14T06:24:49 + return dt.datetime.strptime(value, "%Y-%m-%dT%H:%M:%S").replace(tzinfo=UTC) + + +def build_manifest(root: Path, records: Iterable[Dict[str, Any]], manifest_path: Path) -> None: + manifest_entries = [] + for record in records: + path = record["path"] + rel_path = PurePosixPath(path.relative_to(root).as_posix()) + sha256 = hashlib.sha256(path.read_bytes()).hexdigest() + size = path.stat().st_size + + entry = { + "source": record["source"], + "type": record["type"], + "path": str(rel_path), + "sha256": sha256, + "sizeBytes": size, + "capturedAt": record["capturedAt"], + "from": record.get("from"), + "to": record.get("to"), + "itemCount": record.get("itemCount"), + } + manifest_entries.append(entry) + + sha_file = path.with_suffix(path.suffix + ".sha256") + _write_sha_file(sha_file, sha256, path.name) + + manifest_entries.sort(key=lambda item: item["path"]) + + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_document = { + "source": "feedser.cert-bund", + "generatedAt": dt.datetime.now(tz=UTC).isoformat(), + "artifacts": manifest_entries, + } + + with manifest_path.open("w", encoding="utf-8") as handle: + json.dump(manifest_document, handle, ensure_ascii=False, indent=2, sort_keys=True) + handle.write("\n") + + manifest_sha = hashlib.sha256(manifest_path.read_bytes()).hexdigest() + _write_sha_file(manifest_path.with_suffix(".sha256"), manifest_sha, manifest_path.name) + + print(f"[certbund] manifest generated → {manifest_path}") + + +def _write_sha_file(path: Path, digest: str, filename: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + handle.write(f"{digest} {filename}\n") + + +def parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Capture CERT-Bund search/export snapshots for Offline Kit packaging.", + ) + parser.add_argument("--output", default="seed-data/cert-bund", help="Destination directory for artefacts.") + parser.add_argument("--start-year", type=int, default=2014, help="First year (inclusive) for export snapshots.") + parser.add_argument( + "--end-year", + type=int, + default=dt.datetime.now(tz=UTC).year, + help="Last year (inclusive) for export snapshots.", + ) + parser.add_argument("--page-size", type=int, default=100, help="Search page size.") + parser.add_argument("--max-pages", type=int, default=12, help="Maximum number of search result pages to capture.") + parser.add_argument("--cookie-file", type=Path, help="Path to a Netscape cookie file to reuse/persist session cookies.") + parser.add_argument("--xsrf-token", help="Optional explicit XSRF token value (overrides cookie discovery).") + parser.add_argument( + "--skip-fetch", + action="store_true", + help="Skip HTTP fetches and only regenerate manifest from existing files.", + ) + parser.add_argument( + "--no-bootstrap", + action="store_true", + help="Do not attempt automatic session bootstrap (use with --skip-fetch or pre-populated cookies).", + ) + return parser.parse_args(argv) + + +def main(argv: Optional[List[str]] = None) -> int: + args = parse_args(argv) + output_dir = Path(args.output).expanduser().resolve() + + if not args.skip_fetch: + client = CertBundClient( + cookie_file=args.cookie_file, + xsrf_token=args.xsrf_token, + auto_bootstrap=not args.no_bootstrap, + ) + + start_year = args.start_year + end_year = args.end_year + if start_year > end_year: + raise SystemExit("start-year cannot be greater than end-year.") + + client.fetch_search_pages(output_dir / "search", args.page_size, args.max_pages) + client.fetch_exports(output_dir / "export", start_year, end_year) + + records = scan_artifacts(output_dir) + if not records: + print( + "[certbund] no artefacts discovered. Fetch data first or point --output to the dataset directory.", + file=sys.stderr, + ) + return 1 + + manifest_path = output_dir / "manifest" / "certbund-offline-manifest.json" + build_manifest(output_dir, records, manifest_path) + return 0 + + +if __name__ == "__main__": + sys.exit(main())