Compare commits

...

9 Commits

1521 changed files with 55895 additions and 10340 deletions

8
.gitignore vendored
View File

@@ -18,4 +18,10 @@ obj/
*.log
TestResults/
.dotnet
.dotnet
.DS_Store
seed-data/ics-cisa/*.csv
seed-data/ics-cisa/*.xlsx
seed-data/ics-cisa/*.sha256
seed-data/cert-bund/**/*.json
seed-data/cert-bund/**/*.sha256

View File

@@ -10,7 +10,7 @@ Licence audit identifies potential conflicts, especially copyleft obligations.
Misconfiguration checks detect unsafe Dockerfile patterns (root user, latest tags, permissive modes).
Provenance features include in-toto/SLSA attestations signed with cosign for supply-chain trust.
| Guiding principle | What it means for Feedser |
| Guiding principle | What it means for Concelier |
|-------------------|---------------------------|
| **SBOM-first ingest** | Prefer signed SBOMs or reproducible layer diffs before falling back to raw scraping; connectors treat source docs as provenance, never as mutable truth. |
| **Deterministic outputs** | Same inputs yield identical canonical advisories and exported JSON/Trivy DB artefacts; merge hashes and export manifests are reproducible across machines. |
@@ -41,14 +41,14 @@ All modules are contained by one or more projects. Each project goes in its dedi
# 4) Modules
StellaOps is contained by different modules installable via docker containers
- Feedser. Responsible for aggregation and delivery of vulnerability database
- Concelier. Responsible for aggregation and delivery of vulnerability database
- Cli. Command line tool to unlock full potential - request database operations, install scanner, request scan, configure backend
- Backend. Configures and Manages scans
- UI. UI to access the backend (and scanners)
- Agent. Installable daemon that does the scanning
- Zastava. Realtime monitor for allowed (verified) installations.
## 4.1) Feedser
## 4.1) Concelier
It is webservice based module that is responsible for aggregating vulnerabilities information from various sources, parsing and normalizing them into a canonical shape, merging and deduplicating the results in one place, with export capabilities to Json and TrivyDb. It supports init and resume for all of the sources, parse/normalize and merge/deduplication operations, plus export. Export supports delta exports—similarly to full and incremential database backups.
### 4.1.1) Usage
@@ -65,7 +65,7 @@ api available on https://db.stella-ops.org
4. **Export**: JSON tree and/or Trivy DB; package and (optionally) push; write export state.
### 4.1.3) Architecture
For more information of the architecture see `./docs/ARCHITECTURE_FEEDSER.md`.
For more information of the architecture see `./docs/ARCHITECTURE_CONCELIER.md`.
---
@@ -122,7 +122,7 @@ You main characteristics:
- **Coordination**: In case task is discovered as blocked on other team or task, according TASKS.md files that dependency is on needs to be changed by adding new tasks describing the requirement. the current task must be updated as completed. In case task changes, scope or requirements or rules - other documentations needs be updated accordingly.
- **Sprint synchronization**: When given task seek for relevant directory to work on from SPRINTS.md. Confirm its state on both SPRINTS.md and the relevant TODOS.md file. Always check the AGENTS.md in the relevant TODOS.md directory.
- **Tests**: Add/extend fixtures and unit tests per change; never regress determinism or precedence.
- **Test layout**: Use module-specific projects in `StellaOps.Feedser.<Component>.Tests`; shared fixtures/harnesses live in `StellaOps.Feedser.Testing`.
- **Test layout**: Use module-specific projects in `StellaOps.Concelier.<Component>.Tests`; shared fixtures/harnesses live in `StellaOps.Concelier.Testing`.
- **Execution autonomous**: In case you need to continue with more than one options just continue sequentially, unless the continue requires design decision.
---

View File

@@ -1,28 +1,34 @@
# StellaOps Feedser & CLI
# StellaOps Concelier & CLI
This repository hosts the StellaOps Feedser service, its plug-in ecosystem, and the
first-party CLI (`stellaops-cli`). Feedser ingests vulnerability advisories from
This repository hosts the StellaOps Concelier service, its plug-in ecosystem, and the
first-party CLI (`stellaops-cli`). Concelier ingests vulnerability advisories from
authoritative sources, stores them in MongoDB, and exports deterministic JSON and
Trivy DB artefacts. The CLI drives scanner distribution, scan execution, and job
control against the Feedser API.
control against the Concelier API.
## Quickstart
1. Prepare a MongoDB instance and (optionally) install `trivy-db`/`oras`.
2. Copy `etc/feedser.yaml.sample` to `etc/feedser.yaml` and update the storage + telemetry
2. Copy `etc/concelier.yaml.sample` to `etc/concelier.yaml` and update the storage + telemetry
settings.
3. Copy `etc/authority.yaml.sample` to `etc/authority.yaml`, review the issuer, token
lifetimes, and plug-in descriptors, then edit the companion manifests under
`etc/authority.plugins/*.yaml` to match your deployment.
4. Start the web service with `dotnet run --project src/StellaOps.Feedser.WebService`.
4. Start the web service with `dotnet run --project src/StellaOps.Concelier.WebService`.
5. Configure the CLI via environment variables (e.g. `STELLAOPS_BACKEND_URL`) and trigger
jobs with `dotnet run --project src/StellaOps.Cli -- db merge`.
Detailed operator guidance is available in `docs/10_FEEDSER_CLI_QUICKSTART.md`. API and
Detailed operator guidance is available in `docs/10_CONCELIER_CLI_QUICKSTART.md`. API and
command reference material lives in `docs/09_API_CLI_REFERENCE.md`.
Pipeline note: deployment workflows should template `etc/feedser.yaml` during CI/CD,
Pipeline note: deployment workflows should template `etc/concelier.yaml` during CI/CD,
injecting environment-specific Mongo credentials and telemetry endpoints. Upcoming
releases will add Microsoft OAuth (Entra ID) authentication support—track the quickstart
for integration steps once available.
## Documentation
- `docs/README.md` now consolidates the platform index and points to the updated high-level architecture.
- Module architecture dossiers live under `docs/ARCHITECTURE_*.md`; the most relevant here are `docs/ARCHITECTURE_CONCELIER.md` (service layout, merge engine, exports) and `docs/ARCHITECTURE_CLI.md` (command surface, AOT packaging, auth flows). Related services such as the Signer, Attestor, Authority, Scanner, UI, Excititor, Zastava, and DevOps pipeline each have their own dossier.
- Offline operation guidance moved to `docs/24_OFFLINE_KIT.md`, which details bundle composition, verification, and delta workflows. Concelier-specific connector operations stay in `docs/ops/concelier-certbund-operations.md` and companion runbooks under `docs/ops/`.

View File

@@ -1,95 +1,156 @@
| Sprint | Theme | Tasks File Path | Status | Type of Specialist | Task ID | Task Description |
| --- | --- | --- | --- | --- | --- | --- |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-001 | SemVer primitive range-style metadata<br>Instructions to work:<br>DONE Read ./AGENTS.md and src/StellaOps.Feedser.Models/AGENTS.md. This task lays the groundwork—complete the SemVer helper updates before teammates pick up FEEDMODELS-SCHEMA-01-002/003 and FEEDMODELS-SCHEMA-02-900. Use ./src/FASTER_MODELING_AND_NORMALIZATION.md for the target rule structure. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-002 | Provenance decision rationale field<br>Instructions to work:<br>AdvisoryProvenance now carries `decisionReason` and docs/tests were updated. Connectors and merge tasks should populate the field when applying precedence/freshness/tie-breaker logic; see src/StellaOps.Feedser.Models/PROVENANCE_GUIDELINES.md for usage guidance. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-003 | Normalized version rules collection<br>Instructions to work:<br>`AffectedPackage.NormalizedVersions` and supporting comparer/docs/tests shipped. Connector owners must emit rule arrays per ./src/FASTER_MODELING_AND_NORMALIZATION.md and report progress via FEEDMERGE-COORD-02-900 so merge/storage backfills can proceed. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-02-900 | Range primitives for SemVer/EVR/NEVRA metadata<br>Instructions to work:<br>DONE Read ./AGENTS.md and src/StellaOps.Feedser.Models/AGENTS.md before resuming this stalled effort. Confirm helpers align with the new `NormalizedVersions` representation so connectors finishing in Sprint 2 can emit consistent metadata. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Normalization/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter<br>Shared `SemVerRangeRuleBuilder` now outputs primitives + normalized rules per `FASTER_MODELING_AND_NORMALIZATION.md`; CVE/GHSA connectors consuming the API have verified fixtures. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill<br>AdvisoryStore dual-writes flattened `normalizedVersions` when `feedser.storage.enableSemVerStyle` is set; migration `20251011-semver-style-backfill` updates historical records and docs outline the rollout. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence<br>Storage now persists `provenance.decisionReason` for advisories and merge events; tests cover round-trips. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing<br>Bootstrapper seeds compound/sparse indexes for flattened normalized rules and `docs/dev/mongo_indices.md` documents query guidance. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor<br>Updated constructors/tests keep storage suites passing with the new feature flag defaults. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options<br>WebService wires `authority.resilience.*` into `AddStellaOpsAuthClient` and adds binding coverage via `AuthorityClientResilienceOptionsAreBound`. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning<br>Install/runbooks document connected vs air-gapped resilience profiles and monitoring hooks. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns<br>Operator guides now call out `route/status/subject/clientId/scopes/bypass/remote` audit fields and SIEM triggers. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Feedser operator guide for enforcement cutoff<br>Install guide reiterates the 2025-12-31 cutoff and links audit signals to the rollout checklist. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-001 | SemVer primitive range-style metadata<br>Instructions to work:<br>DONE Read ./AGENTS.md and src/StellaOps.Concelier.Models/AGENTS.md. This task lays the groundwork—complete the SemVer helper updates before teammates pick up FEEDMODELS-SCHEMA-01-002/003 and FEEDMODELS-SCHEMA-02-900. Use ./src/FASTER_MODELING_AND_NORMALIZATION.md for the target rule structure. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-002 | Provenance decision rationale field<br>Instructions to work:<br>AdvisoryProvenance now carries `decisionReason` and docs/tests were updated. Connectors and merge tasks should populate the field when applying precedence/freshness/tie-breaker logic; see src/StellaOps.Concelier.Models/PROVENANCE_GUIDELINES.md for usage guidance. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-003 | Normalized version rules collection<br>Instructions to work:<br>`AffectedPackage.NormalizedVersions` and supporting comparer/docs/tests shipped. Connector owners must emit rule arrays per ./src/FASTER_MODELING_AND_NORMALIZATION.md and report progress via FEEDMERGE-COORD-02-900 so merge/storage backfills can proceed. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-02-900 | Range primitives for SemVer/EVR/NEVRA metadata<br>Instructions to work:<br>DONE Read ./AGENTS.md and src/StellaOps.Concelier.Models/AGENTS.md before resuming this stalled effort. Confirm helpers align with the new `NormalizedVersions` representation so connectors finishing in Sprint 2 can emit consistent metadata. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Normalization/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter<br>Shared `SemVerRangeRuleBuilder` now outputs primitives + normalized rules per `FASTER_MODELING_AND_NORMALIZATION.md`; CVE/GHSA connectors consuming the API have verified fixtures. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill<br>AdvisoryStore dual-writes flattened `normalizedVersions` when `concelier.storage.enableSemVerStyle` is set; migration `20251011-semver-style-backfill` updates historical records and docs outline the rollout. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence<br>Storage now persists `provenance.decisionReason` for advisories and merge events; tests cover round-trips. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing<br>Bootstrapper seeds compound/sparse indexes for flattened normalized rules and `docs/dev/mongo_indices.md` documents query guidance. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor<br>Updated constructors/tests keep storage suites passing with the new feature flag defaults. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options<br>WebService wires `authority.resilience.*` into `AddStellaOpsAuthClient` and adds binding coverage via `AuthorityClientResilienceOptionsAreBound`. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning<br>Install/runbooks document connected vs air-gapped resilience profiles and monitoring hooks. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns<br>Operator guides now call out `route/status/subject/clientId/scopes/bypass/remote` audit fields and SIEM triggers. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Concelier operator guide for enforcement cutoff<br>Install guide reiterates the 2025-12-31 cutoff and links audit signals to the rollout checklist. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | SEC3.HOST | Rate limiter policy binding<br>Authority host now applies configuration-driven fixed windows to `/token`, `/authorize`, and `/internal/*`; integration tests assert 429 + `Retry-After` headers; docs/config samples refreshed for Docs guild diagrams. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | SEC3.BUILD | Authority rate-limiter follow-through<br>`Security.RateLimiting` now fronts token/authorize/internal limiters; Authority + Configuration matrices (`dotnet test src/StellaOps.Authority/StellaOps.Authority.sln`, `dotnet test src/StellaOps.Configuration.Tests/StellaOps.Configuration.Tests.csproj`) passed on 2025-10-11; awaiting #authority-core broadcast. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-14) | Team Authority Platform & Security Guild | AUTHCORE-BUILD-OPENIDDICT / AUTHCORE-STORAGE-DEVICE-TOKENS / AUTHCORE-BOOTSTRAP-INVITES | Address remaining Authority compile blockers (OpenIddict transaction shim, token device document, bootstrap invite cleanup) so `dotnet build src/StellaOps.Authority.sln` returns success. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | PLG6.DOC | Plugin developer guide polish<br>Section 9 now documents rate limiter metadata, config keys, and lockout interplay; YAML samples updated alongside Authority config templates. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-001 | Fetch pipeline & state tracking<br>Summary planner now drives monthly/yearly VINCE fetches, persists pending summaries/notes, and hydrates VINCE detail queue with telemetry.<br>Team instructions: Read ./AGENTS.md and src/StellaOps.Feedser.Source.CertCc/AGENTS.md. Coordinate daily with Models/Merge leads so new normalizedVersions output and provenance tags stay aligned with ./src/FASTER_MODELING_AND_NORMALIZATION.md. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-002 | VINCE note detail fetcher<br>Summary planner queues VINCE note detail endpoints, persists raw JSON with SHA/ETag metadata, and records retry/backoff metrics. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-003 | DTO & parser implementation<br>Added VINCE DTO aggregate, Markdown→text sanitizer, vendor/status/vulnerability parsers, and parser regression fixture. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-004 | Canonical mapping & range primitives<br>VINCE DTO aggregate flows through `CertCcMapper`, emitting vendor range primitives + normalized version rules that persist via `_advisoryStore`. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-005 | Deterministic fixtures/tests<br>Snapshot harness refreshed 2025-10-12; `certcc-*.snapshot.json` regenerated and regression suite green without UPDATE flag drift. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-006 | Telemetry & documentation<br>`CertCcDiagnostics` publishes summary/detail/parse/map metrics (meter `StellaOps.Feedser.Source.CertCc`), README documents instruments, and log guidance captured for Ops on 2025-10-12. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-007 | Connector test harness remediation<br>Harness now wires `AddSourceCommon`, resets `FakeTimeProvider`, and passes canned-response regression run dated 2025-10-12. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | BLOCKED (2025-10-11) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-008 | Snapshot coverage handoff<br>Upstream repo version lacks SemVer primitives + provenance decision reason fields, so snapshot regeneration fails; resume once Models/Storage sprint lands those changes. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-012 | Schema sync & snapshot regen follow-up<br>Fixtures regenerated with normalizedVersions + provenance decision reasons; handoff notes updated for Merge backfill 2025-10-12. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-009 | Detail/map reintegration plan<br>Staged reintegration plan published in `src/StellaOps.Feedser.Source.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md`; coordinates enablement with FEEDCONN-CERTCC-02-004. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-010 | Partial-detail graceful degradation<br>Detail fetch now tolerates 404/403/410 responses and regression tests cover mixed endpoint availability. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Distro.RedHat/TASKS.md | DONE (2025-10-11) | Team Connector Resumption CERT/RedHat | FEEDCONN-REDHAT-02-001 | Fixture validation sweep<br>Instructions to work:<br>Fixtures regenerated post-model-helper rollout; provenance ordering and normalizedVersions scaffolding verified via tests. Conflict resolver deltas logged in src/StellaOps.Feedser.Source.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md for Sprint 3 consumers. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-001 | Canonical mapping & range primitives<br>Mapper emits SemVer rules (`scheme=apple:*`); fixtures regenerated with trimmed references + new RSR coverage, update tooling finalized. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-002 | Deterministic fixtures/tests<br>Sanitized live fixtures + regression snapshots wired into tests; normalized rule coverage asserted. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-003 | Telemetry & documentation<br>Apple meter metrics wired into Feedser WebService OpenTelemetry configuration; README and fixtures document normalizedVersions coverage. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-004 | Live HTML regression sweep<br>Sanitised HT125326/HT125328/HT106355/HT214108/HT215500 fixtures recorded and regression tests green on 2025-10-12. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-005 | Fixture regeneration tooling<br>`UPDATE_APPLE_FIXTURES=1` flow fetches & rewrites fixtures; README documents usage.<br>Instructions to work:<br>DONE Read ./AGENTS.md and src/StellaOps.Feedser.Source.Vndr.Apple/AGENTS.md. Resume stalled tasks, ensuring normalizedVersions output and fixtures align with ./src/FASTER_MODELING_AND_NORMALIZATION.md before handing data to the conflict sprint. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance<br>Team instructions: Read ./AGENTS.md and each module's AGENTS file. Adopt the `NormalizedVersions` array emitted by the models sprint, wiring provenance `decisionReason` where merge overrides occur. Follow ./src/FASTER_MODELING_AND_NORMALIZATION.md; report via src/StellaOps.Feedser.Merge/TASKS.md (FEEDMERGE-COORD-02-900). Progress 2025-10-11: GHSA/OSV emit normalized arrays with refreshed fixtures; CVE mapper now surfaces SemVer normalized ranges; NVD/KEV adoption pending; outstanding follow-ups include FEEDSTORAGE-DATA-02-001, FEEDMERGE-ENGINE-02-002, and rolling `tools/FixtureUpdater` updates across connectors. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Cve/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-CVE-02-003 | CVE normalized versions uplift |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Kev/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-KEV-02-003 | KEV normalized versions propagation |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-04-003 | OSV parity fixture refresh |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-10) | Team WebService & Authority | FEEDWEB-DOCS-01-001 | Document authority toggle & scope requirements<br>Quickstart carries toggle/scope guidance pending docs guild review (no change this sprint). |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options<br>WebService wires `authority.resilience.*` into `AddStellaOpsAuthClient` and adds binding coverage via `AuthorityClientResilienceOptionsAreBound`. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning<br>Operator docs now outline connected vs air-gapped resilience profiles and monitoring cues. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns<br>Audit logging guidance highlights `route/status/subject/clientId/scopes/bypass/remote` fields and SIEM alerts. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Feedser operator guide for enforcement cutoff<br>Install guide reiterates the 2025-12-31 cutoff and ties audit signals to rollout checks. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-OPS-01-006 | Rename plugin drop directory to namespaced path<br>Build outputs, tests, and docs now target `StellaOps.Feedser.PluginBinaries`/`StellaOps.Authority.PluginBinaries`. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-OPS-01-007 | Authority resilience adoption<br>Deployment docs and CLI notes explain the LIB5 resilience knobs for rollout.<br>Instructions to work:<br>DONE Read ./AGENTS.md and src/StellaOps.Feedser.WebService/AGENTS.md. These items were mid-flight; resume implementation ensuring docs/operators receive timely updates. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-001 | Fetch pipeline & state tracking<br>Summary planner now drives monthly/yearly VINCE fetches, persists pending summaries/notes, and hydrates VINCE detail queue with telemetry.<br>Team instructions: Read ./AGENTS.md and src/StellaOps.Concelier.Connector.CertCc/AGENTS.md. Coordinate daily with Models/Merge leads so new normalizedVersions output and provenance tags stay aligned with ./src/FASTER_MODELING_AND_NORMALIZATION.md. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-002 | VINCE note detail fetcher<br>Summary planner queues VINCE note detail endpoints, persists raw JSON with SHA/ETag metadata, and records retry/backoff metrics. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-003 | DTO & parser implementation<br>Added VINCE DTO aggregate, Markdown→text sanitizer, vendor/status/vulnerability parsers, and parser regression fixture. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-004 | Canonical mapping & range primitives<br>VINCE DTO aggregate flows through `CertCcMapper`, emitting vendor range primitives + normalized version rules that persist via `_advisoryStore`. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-005 | Deterministic fixtures/tests<br>Snapshot harness refreshed 2025-10-12; `certcc-*.snapshot.json` regenerated and regression suite green without UPDATE flag drift. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-006 | Telemetry & documentation<br>`CertCcDiagnostics` publishes summary/detail/parse/map metrics (meter `StellaOps.Concelier.Connector.CertCc`), README documents instruments, and log guidance captured for Ops on 2025-10-12. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-007 | Connector test harness remediation<br>Harness now wires `AddSourceCommon`, resets `FakeTimeProvider`, and passes canned-response regression run dated 2025-10-12. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-008 | Snapshot coverage handoff<br>Fixtures regenerated with normalized ranges + provenance fields on 2025-10-11; QA handoff notes published and merge backfill unblocked. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-012 | Schema sync & snapshot regen follow-up<br>Fixtures regenerated with normalizedVersions + provenance decision reasons; handoff notes updated for Merge backfill 2025-10-12. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-009 | Detail/map reintegration plan<br>Staged reintegration plan published in `src/StellaOps.Concelier.Connector.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md`; coordinates enablement with FEEDCONN-CERTCC-02-004. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption CERT/RedHat | FEEDCONN-CERTCC-02-010 | Partial-detail graceful degradation<br>Detail fetch now tolerates 404/403/410 responses and regression tests cover mixed endpoint availability. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Distro.RedHat/TASKS.md | DONE (2025-10-11) | Team Connector Resumption CERT/RedHat | FEEDCONN-REDHAT-02-001 | Fixture validation sweep<br>Instructions to work:<br>Fixtures regenerated post-model-helper rollout; provenance ordering and normalizedVersions scaffolding verified via tests. Conflict resolver deltas logged in src/StellaOps.Concelier.Connector.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md for Sprint 3 consumers. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-001 | Canonical mapping & range primitives<br>Mapper emits SemVer rules (`scheme=apple:*`); fixtures regenerated with trimmed references + new RSR coverage, update tooling finalized. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-002 | Deterministic fixtures/tests<br>Sanitized live fixtures + regression snapshots wired into tests; normalized rule coverage asserted. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-003 | Telemetry & documentation<br>Apple meter metrics wired into Concelier WebService OpenTelemetry configuration; README and fixtures document normalizedVersions coverage. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-004 | Live HTML regression sweep<br>Sanitised HT125326/HT125328/HT106355/HT214108/HT215500 fixtures recorded and regression tests green on 2025-10-12. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-005 | Fixture regeneration tooling<br>`UPDATE_APPLE_FIXTURES=1` flow fetches & rewrites fixtures; README documents usage.<br>Instructions to work:<br>DONE Read ./AGENTS.md and src/StellaOps.Concelier.Connector.Vndr.Apple/AGENTS.md. Resume stalled tasks, ensuring normalizedVersions output and fixtures align with ./src/FASTER_MODELING_AND_NORMALIZATION.md before handing data to the conflict sprint. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance<br>Team instructions: Read ./AGENTS.md and each module's AGENTS file. Adopt the `NormalizedVersions` array emitted by the models sprint, wiring provenance `decisionReason` where merge overrides occur. Follow ./src/FASTER_MODELING_AND_NORMALIZATION.md; report via src/StellaOps.Concelier.Merge/TASKS.md (FEEDMERGE-COORD-02-900). Progress 2025-10-11: GHSA/OSV emit normalized arrays with refreshed fixtures; CVE mapper now surfaces SemVer normalized ranges; NVD/KEV adoption pending; outstanding follow-ups include FEEDSTORAGE-DATA-02-001, FEEDMERGE-ENGINE-02-002, and rolling `tools/FixtureUpdater` updates across connectors. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Cve/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-CVE-02-003 | CVE normalized versions uplift |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Kev/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-KEV-02-003 | KEV normalized versions propagation |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-04-003 | OSV parity fixture refresh |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-10) | Team WebService & Authority | FEEDWEB-DOCS-01-001 | Document authority toggle & scope requirements<br>Quickstart carries toggle/scope guidance pending docs guild review (no change this sprint). |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options<br>WebService wires `authority.resilience.*` into `AddStellaOpsAuthClient` and adds binding coverage via `AuthorityClientResilienceOptionsAreBound`. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning<br>Operator docs now outline connected vs air-gapped resilience profiles and monitoring cues. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns<br>Audit logging guidance highlights `route/status/subject/clientId/scopes/bypass/remote` fields and SIEM alerts. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Concelier operator guide for enforcement cutoff<br>Install guide reiterates the 2025-12-31 cutoff and ties audit signals to rollout checks. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-OPS-01-006 | Rename plugin drop directory to namespaced path<br>Build outputs, tests, and docs now target `StellaOps.Concelier.PluginBinaries`/`StellaOps.Authority.PluginBinaries`. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-OPS-01-007 | Authority resilience adoption<br>Deployment docs and CLI notes explain the LIB5 resilience knobs for rollout.<br>Instructions to work:<br>DONE Read ./AGENTS.md and src/StellaOps.Concelier.WebService/AGENTS.md. These items were mid-flight; resume implementation ensuring docs/operators receive timely updates. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHCORE-ENGINE-01-001 | CORE8.RL — Rate limiter plumbing validated; integration tests green and docs handoff recorded for middleware ordering + Retry-After headers (see `docs/dev/authority-rate-limit-tuning-outline.md` for continuing guidance). |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHCRYPTO-ENGINE-01-001 | SEC3.A — Shared metadata resolver confirmed via host test run; SEC3.B now unblocked for tuning guidance (outline captured in `docs/dev/authority-rate-limit-tuning-outline.md`). |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DOING (2025-10-11) | Team Authority Platform & Security Guild | AUTHPLUG-DOCS-01-001 | PLG6.DOC — Docs guild resuming diagram/copy updates using the captured limiter context + configuration notes (reference `docs/dev/authority-rate-limit-tuning-outline.md` for tuning matrix + observability copy).<br>Instructions to work:<br>Read ./AGENTS.md plus module-specific AGENTS. Restart the blocked rate-limiter workstream (Authority host + cryptography) so the plugin docs team can finish diagrams. Coordinate daily; use ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md where rate limiting interacts with conflict policy. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Normalization/TASKS.md | — | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter<br>Instructions to work:<br>Read ./AGENTS.md and module AGENTS. Use ./src/FASTER_MODELING_AND_NORMALIZATION.md to build the shared rule generator; sync daily with storage and connector owners. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | — | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | — | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | — | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing<br>Instructions to work:<br>Read ./AGENTS.md and storage AGENTS. Implement dual-write/backfill and index creation using the shapes from ./src/FASTER_MODELING_AND_NORMALIZATION.md; coordinate with connectors entering the sprint. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Merge/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDMERGE-ENGINE-02-002 | Normalized versions union & dedupe<br>Affected package resolver unions/dedupes normalized rules, stamps merge provenance with `decisionReason`, and tests cover the rollout. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-GHSA-02-004 | GHSA credits & ecosystem severity mapping |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | TODO | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-GHSA-02-005 | GitHub quota monitoring & retries |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | TODO | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-GHSA-02-006 | Production credential & scheduler rollout |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-GHSA-02-007 | Credit parity regression fixtures |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-NVD-02-004 | NVD CVSS & CWE precedence payloads |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-NVD-02-005 | NVD merge/export parity regression |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-OSV-02-004 | OSV references & credits alignment |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-OSV-02-005 | Fixture updater workflow<br>Resolved 2025-10-12: OSV mapper now derives canonical PURLs for Go + scoped npm packages when raw payloads omit `purl`; conflict fixtures unchanged for invalid npm names. Verified via `dotnet test src/StellaOps.Feedser.Source.Osv.Tests`, `src/StellaOps.Feedser.Source.Ghsa.Tests`, `src/StellaOps.Feedser.Source.Nvd.Tests`, and backbone normalization/storage suites. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Acsc/TASKS.md | Implementation DOING | Team Connector Expansion Regional & Vendor Feeds | FEEDCONN-ACSC-02-001 … 02-008 | Fetch pipeline, DTO parser, canonical mapper, fixtures, and README shipped 2025-10-12; downstream export integration still pending future tasks. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Cccs/TASKS.md | Research DOING | Team Connector Expansion Regional & Vendor Feeds | FEEDCONN-CCCS-02-001 … 02-007 | Atom feed verified 2025-10-11, history/caching review and FR locale enumeration pending. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.CertBund/TASKS.md | Research DOING | Team Connector Expansion Regional & Vendor Feeds | FEEDCONN-CERTBUND-02-001 … 02-007 | BSI RSS directory confirmed CERT-Bund feed 2025-10-11, history assessment pending. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Kisa/TASKS.md | Research DOING | Team Connector Expansion Regional & Vendor Feeds | FEEDCONN-KISA-02-001 … 02-007 | KNVD RSS endpoint identified 2025-10-11, access headers/session strategy outstanding. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ru.Bdu/TASKS.md | Build DOING | Team Connector Expansion Regional & Vendor Feeds | FEEDCONN-RUBDU-02-001 … 02-008 | TLS bundle + connectors landed 2025-10-12; fetch/parse/map flow emits advisories, fixtures & telemetry follow-up pending. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md | Build DOING | Team Connector Expansion Regional & Vendor Feeds | FEEDCONN-NKCKI-02-001 … 02-008 | JSON bulletin fetch + canonical mapping live 2025-10-12; regression fixtures added but blocked on Mongo2Go libcrypto dependency for test execution. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ics.Cisa/TASKS.md | Research DOING | Team Connector Expansion Regional & Vendor Feeds | FEEDCONN-ICSCISA-02-001 … 02-008 | new ICS RSS endpoint logged 2025-10-11 but Akamai blocks direct pulls, fallback strategy task opened. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Vndr.Cisco/TASKS.md | Research DOING | Team Connector Expansion Regional & Vendor Feeds | FEEDCONN-CISCO-02-001 … 02-007 | openVuln API + RSS reviewed 2025-10-11, auth/pagination memo pending. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Vndr.Msrc/TASKS.md | Research DOING | Team Connector Expansion Regional & Vendor Feeds | FEEDCONN-MSRC-02-001 … 02-007 | MSRC API docs reviewed 2025-10-11, auth/throttling comparison memo pending.<br>Instructions to work:<br>Read ./AGENTS.md plus each module's AGENTS file. Parallelize research, ingestion, mapping, fixtures, and docs using the normalized rule shape from ./src/FASTER_MODELING_AND_NORMALIZATION.md. Coordinate daily with the merge coordination task from Sprint 1. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Cve/TASKS.md | — | Team Connector Support & Monitoring | FEEDCONN-CVE-02-001 … 02-002 | Instructions to work:<br>Read ./AGENTS.md and module AGENTS. Deliver operator docs and monitoring instrumentation required for broader feed rollout. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Kev/TASKS.md | — | Team Connector Support & Monitoring | FEEDCONN-KEV-02-001 … 02-002 | Instructions to work:<br>Read ./AGENTS.md and module AGENTS. Deliver operator docs and monitoring instrumentation required for broader feed rollout. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-13) | Team Authority Platform & Security Guild | AUTHSEC-DOCS-01-002 | SEC3.B — Published `docs/security/rate-limits.md` with tuning matrix, alert thresholds, and lockout interplay guidance; Docs guild can lift copy into plugin guide. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-14) | Team Authority Platform & Security Guild | AUTHSEC-CRYPTO-02-001 | SEC5.B1 — Introduce libsodium signing provider and parity tests to unblock CLI verification enhancements. |
| Sprint 1 | Bootstrap & Replay Hardening | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-14) | Security Guild | AUTHSEC-CRYPTO-02-004 | SEC5.D/E — Finish bootstrap invite lifecycle (API/store/cleanup) and token device heuristics; build currently red due to pending handler integration. |
| Sprint 1 | Developer Tooling | src/StellaOps.Cli/TASKS.md | DONE (2025-10-15) | DevEx/CLI | AUTHCLI-DIAG-01-001 | Surface password policy diagnostics in CLI startup/output so operators see weakened overrides immediately.<br>CLI now loads Authority plug-ins at startup, logs weakened password policies (length/complexity), and regression coverage lives in `StellaOps.Cli.Tests/Services/AuthorityDiagnosticsReporterTests`. |
| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHPLUG-DOCS-01-001 | PLG6.DOC — Developer guide copy + diagrams merged 2025-10-11; limiter guidance incorporated and handed to Docs guild for asset export. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Normalization/TASKS.md | DONE (2025-10-12) | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter<br>`SemVerRangeRuleBuilder` shipped 2025-10-12 with comparator/`||` support and fixtures aligning to `FASTER_MODELING_AND_NORMALIZATION.md`. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing<br>Indexes seeded + docs updated 2025-10-11 to cover flattened normalized rules for connector adoption. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDMERGE-ENGINE-02-002 | Normalized versions union & dedupe<br>Affected package resolver unions/dedupes normalized rules, stamps merge provenance with `decisionReason`, and tests cover the rollout. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-GHSA-02-004 | GHSA credits & ecosystem severity mapping |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-GHSA-02-005 | GitHub quota monitoring & retries |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-GHSA-02-006 | Production credential & scheduler rollout |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-GHSA-02-007 | Credit parity regression fixtures |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-NVD-02-004 | NVD CVSS & CWE precedence payloads |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-NVD-02-005 | NVD merge/export parity regression |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-OSV-02-004 | OSV references & credits alignment |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-OSV-02-005 | Fixture updater workflow<br>Resolved 2025-10-12: OSV mapper now derives canonical PURLs for Go + scoped npm packages when raw payloads omit `purl`; conflict fixtures unchanged for invalid npm names. Verified via `dotnet test src/StellaOps.Concelier.Connector.Osv.Tests`, `src/StellaOps.Concelier.Connector.Ghsa.Tests`, `src/StellaOps.Concelier.Connector.Nvd.Tests`, and backbone normalization/storage suites. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Acsc/TASKS.md | DONE (2025-10-12) | Team Connector Expansion Regional & Vendor Feeds | FEEDCONN-ACSC-02-001 … 02-008 | Fetch→parse→map pipeline, fixtures, diagnostics, and README finished 2025-10-12; downstream export parity captured via FEEDEXPORT-JSON-04-001 / FEEDEXPORT-TRIVY-04-001 (completed). |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Cccs/TASKS.md | DONE (2025-10-16) | Team Connector Expansion Regional & Vendor Feeds | FEEDCONN-CCCS-02-001 … 02-008 | Observability meter, historical harvest plan, and DOM sanitizer refinements wrapped; ops notes live under `docs/ops/concelier-cccs-operations.md` with fixtures validating EN/FR list handling. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.CertBund/TASKS.md | DONE (2025-10-15) | Team Connector Expansion Regional & Vendor Feeds | FEEDCONN-CERTBUND-02-001 … 02-008 | Telemetry/docs (02-006) and history/locale sweep (02-007) completed alongside pipeline; runbook `docs/ops/concelier-certbund-operations.md` captures locale guidance and offline packaging. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Kisa/TASKS.md | DONE (2025-10-14) | Team Connector Expansion Regional & Vendor Feeds | FEEDCONN-KISA-02-001 … 02-007 | Connector, tests, and telemetry/docs (02-006) finalized; localisation notes in `docs/dev/kisa_connector_notes.md` complete rollout. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ru.Bdu/TASKS.md | DONE (2025-10-14) | Team Connector Expansion Regional & Vendor Feeds | FEEDCONN-RUBDU-02-001 … 02-008 | Fetch/parser/mapper refinements, regression fixtures, telemetry/docs, access options, and trusted root packaging all landed; README documents offline access strategy. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ru.Nkcki/TASKS.md | DONE (2025-10-13) | Team Connector Expansion Regional & Vendor Feeds | FEEDCONN-NKCKI-02-001 … 02-008 | Listing fetch, parser, mapper, fixtures, telemetry/docs, and archive plan finished; Mongo2Go/libcrypto dependency resolved via bundled OpenSSL noted in ops guide. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ics.Cisa/TASKS.md | DONE (2025-10-16) | Team Connector Expansion Regional & Vendor Feeds | FEEDCONN-ICSCISA-02-001 … 02-011 | Feed parser attachment fixes, SemVer exact values, regression suites, telemetry/docs updates, and handover complete; ops runbook now details attachment verification + proxy usage. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Vndr.Cisco/TASKS.md | DONE (2025-10-14) | Team Connector Expansion Regional & Vendor Feeds | FEEDCONN-CISCO-02-001 … 02-007 | OAuth fetch pipeline, DTO/mapping, tests, and telemetry/docs shipped; monitoring/export integration follow-ups recorded in Ops docs and exporter backlog (completed). |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Vndr.Msrc/TASKS.md | DONE (2025-10-15) | Team Connector Expansion Regional & Vendor Feeds | FEEDCONN-MSRC-02-001 … 02-008 | Azure AD onboarding (02-008) unblocked fetch/parse/map pipeline; fixtures, telemetry/docs, and Offline Kit guidance published in `docs/ops/concelier-msrc-operations.md`. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Cve/TASKS.md | DONE (2025-10-15) | Team Connector Support & Monitoring | FEEDCONN-CVE-02-001 … 02-002 | CVE data-source selection, fetch pipeline, and docs landed 2025-10-10. 2025-10-15: smoke verified using the seeded mirror fallback; connector now logs a warning and pulls from `seed-data/cve/` until live CVE Services credentials arrive. |
| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Kev/TASKS.md | DONE (2025-10-12) | Team Connector Support & Monitoring | FEEDCONN-KEV-02-001 … 02-002 | KEV catalog ingestion, fixtures, telemetry, and schema validation completed 2025-10-12; ops dashboard published. |
| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-01-001 | Canonical schema docs refresh<br>Updated canonical schema + provenance guides with SemVer style, normalized version rules, decision reason change log, and migration notes. |
| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-02-001 | Feedser-SemVer Playbook<br>Published merge playbook covering mapper patterns, dedupe flow, indexes, and rollout checklist. |
| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-02-001 | Concelier-SemVer Playbook<br>Published merge playbook covering mapper patterns, dedupe flow, indexes, and rollout checklist. |
| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-02-002 | Normalized versions query guide<br>Delivered Mongo index/query addendum with `$unwind` recipes, dedupe checks, and operational checklist.<br>Instructions to work:<br>DONE Read ./AGENTS.md and docs/AGENTS.md. Document every schema/index/query change produced in Sprint 1-2 leveraging ./src/FASTER_MODELING_AND_NORMALIZATION.md. |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-001 | Canonical merger implementation<br>`CanonicalMerger` ships with freshness/tie-breaker logic, provenance, and unit coverage feeding Merge. |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-002 | Field precedence and tie-breaker map<br>Field precedence tables and tie-breaker metrics wired into the canonical merge flow; docs/tests updated.<br>Instructions to work:<br>Read ./AGENTS.md and core AGENTS. Implement the conflict resolver exactly as specified in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md, coordinating with Merge and Storage teammates. |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-03-001 | Merge event provenance audit prep<br>Merge events now persist `fieldDecisions` and analytics-ready provenance snapshots. |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill<br>Dual-write/backfill flag delivered; migration + options validated in tests. |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor<br>Storage tests adjusted for normalized versions/decision reasons.<br>Instructions to work:<br>Read ./AGENTS.md and storage AGENTS. Extend merge events with decision reasons and analytics views to support the conflict rules, and deliver the dual-write/backfill for `NormalizedVersions` + `decisionReason` so connectors can roll out safely. |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-001 | GHSA/NVD/OSV conflict rules<br>Merge pipeline consumes `CanonicalMerger` output prior to precedence merge. |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-002 | Override metrics instrumentation<br>Merge events capture per-field decisions; counters/logs align with conflict rules. |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-003 | Reference & credit union pipeline<br>Canonical merge preserves unions with updated tests. |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-QA-04-001 | End-to-end conflict regression suite<br>Added regression tests (`AdvisoryMergeServiceTests`) covering canonical + precedence flow.<br>Instructions to work:<br>Read ./AGENTS.md and merge AGENTS. Integrate the canonical merger, instrument metrics, and deliver comprehensive regression tests following ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md. |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-002 | GHSA conflict regression fixtures |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Source.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-NVD-04-002 | NVD conflict regression fixtures |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Source.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-OSV-04-002 | OSV conflict regression fixtures<br>Instructions to work:<br>Read ./AGENTS.md and module AGENTS. Produce fixture triples supporting the precedence/tie-breaker paths defined in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md and hand them to Merge QA. |
| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | DONE (2025-10-11) | Team Documentation Guild Conflict Guidance | FEEDDOCS-DOCS-05-001 | Feedser Conflict Rules<br>Runbook published at `docs/ops/feedser-conflict-resolution.md`; metrics/log guidance aligned with Sprint 3 merge counters. |
| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | TODO | Team Documentation Guild Conflict Guidance | FEEDDOCS-DOCS-05-002 | Conflict runbook ops rollout<br>Instructions to work:<br>Read ./AGENTS.md and docs/AGENTS.md. Once GHSA/NVD/OSV regression fixtures (FEEDCONN-GHSA-04-002, FEEDCONN-NVD-04-002, FEEDCONN-OSV-04-002) are delivered, schedule the Ops review, apply the alert thresholds captured in `docs/ops/feedser-authority-audit-runbook.md`, and record change-log linkage after sign-off. Use ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md for ongoing rule references. |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-001 | Canonical merger implementation<br>`CanonicalMerger` ships with freshness/tie-breaker logic, provenance, and unit coverage feeding Merge. |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-002 | Field precedence and tie-breaker map<br>Field precedence tables and tie-breaker metrics wired into the canonical merge flow; docs/tests updated.<br>Instructions to work:<br>Read ./AGENTS.md and core AGENTS. Implement the conflict resolver exactly as specified in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md, coordinating with Merge and Storage teammates. |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-03-001 | Merge event provenance audit prep<br>Merge events now persist `fieldDecisions` and analytics-ready provenance snapshots. |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill<br>Dual-write/backfill flag delivered; migration + options validated in tests. |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor<br>Storage tests adjusted for normalized versions/decision reasons.<br>Instructions to work:<br>Read ./AGENTS.md and storage AGENTS. Extend merge events with decision reasons and analytics views to support the conflict rules, and deliver the dual-write/backfill for `NormalizedVersions` + `decisionReason` so connectors can roll out safely. |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-001 | GHSA/NVD/OSV conflict rules<br>Merge pipeline consumes `CanonicalMerger` output prior to precedence merge. |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-002 | Override metrics instrumentation<br>Merge events capture per-field decisions; counters/logs align with conflict rules. |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-003 | Reference & credit union pipeline<br>Canonical merge preserves unions with updated tests. |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-QA-04-001 | End-to-end conflict regression suite<br>Added regression tests (`AdvisoryMergeServiceTests`) covering canonical + precedence flow.<br>Instructions to work:<br>Read ./AGENTS.md and merge AGENTS. Integrate the canonical merger, instrument metrics, and deliver comprehensive regression tests following ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md. |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-002 | GHSA conflict regression fixtures |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-NVD-04-002 | NVD conflict regression fixtures |
| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-OSV-04-002 | OSV conflict regression fixtures<br>Instructions to work:<br>Read ./AGENTS.md and module AGENTS. Produce fixture triples supporting the precedence/tie-breaker paths defined in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md and hand them to Merge QA. |
| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | DONE (2025-10-11) | Team Documentation Guild Conflict Guidance | FEEDDOCS-DOCS-05-001 | Concelier Conflict Rules<br>Runbook published at `docs/ops/concelier-conflict-resolution.md`; metrics/log guidance aligned with Sprint 3 merge counters. |
| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | DONE (2025-10-16) | Team Documentation Guild Conflict Guidance | FEEDDOCS-DOCS-05-002 | Conflict runbook ops rollout<br>Ops review completed, alert thresholds applied, and change log appended in `docs/ops/concelier-conflict-resolution.md`; task closed after connector signals verified. |
| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-15) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-04-001 | Advisory schema parity (description/CWE/canonical metric)<br>Extend `Advisory` and related records with description text, CWE collection, and canonical metric pointer; refresh validation + serializer determinism tests. |
| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-15) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-04-003 | Canonical merger parity for new fields<br>Teach `CanonicalMerger` to populate description, CWEResults, and canonical metric pointer with provenance + regression coverage. |
| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-15) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-04-004 | Reference normalization & freshness instrumentation cleanup<br>Implement URL normalization for reference dedupe, align freshness-sensitive instrumentation, and add analytics tests. |
| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-15) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-004 | Merge pipeline parity for new advisory fields<br>Ensure merge service + merge events surface description/CWE/canonical metric decisions with updated metrics/tests. |
| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-15) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-005 | Connector coordination for new advisory fields<br>GHSA/NVD/OSV connectors now ship description, CWE, and canonical metric data with refreshed fixtures; merge coordination log updated and exporters notified. |
| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Exporter.Json/TASKS.md | DONE (2025-10-15) | Team Exporters JSON | FEEDEXPORT-JSON-04-001 | Surface new advisory fields in JSON exporter<br>Update schemas/offline bundle + fixtures once model/core parity lands.<br>2025-10-15: `dotnet test src/StellaOps.Concelier.Exporter.Json.Tests` validated canonical metric/CWE emission. |
| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Exporter.TrivyDb/TASKS.md | DONE (2025-10-15) | Team Exporters Trivy DB | FEEDEXPORT-TRIVY-04-001 | Propagate new advisory fields into Trivy DB package<br>Extend Bolt builder, metadata, and regression tests for the expanded schema.<br>2025-10-15: `dotnet test src/StellaOps.Concelier.Exporter.TrivyDb.Tests` confirmed canonical metric/CWE propagation. |
| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-16) | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-004 | Harden CVSS fallback so canonical metric ids persist when GitHub omits vectors; extend fixtures and document severity precedence hand-off to Merge. |
| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-16) | Team Connector Expansion GHSA/NVD/OSV | FEEDCONN-OSV-04-005 | Map OSV advisories lacking CVSS vectors to canonical metric ids/notes and document CWE provenance quirks; schedule parity fixture updates. |
| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-001 | Stand up canonical VEX claim/consensus records with deterministic serializers so Storage/Exports share a stable contract. |
| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-002 | Implement trust-weighted consensus resolver with baseline policy weights, justification gates, telemetry output, and majority/tie handling. |
| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-003 | Publish shared connector/exporter/attestation abstractions and deterministic query signature utilities for cache/attestation workflows. |
| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-15) | Team Excititor Policy | EXCITITOR-POLICY-01-001 | Established policy options & snapshot provider covering baseline weights/overrides. |
| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-15) | Team Excititor Policy | EXCITITOR-POLICY-01-002 | Policy evaluator now feeds consensus resolver with immutable snapshots. |
| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-003 | Author policy diagnostics, CLI/WebService surfacing, and documentation updates. |
| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-004 | Implement YAML/JSON schema validation and deterministic diagnostics for operator bundles. |
| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-005 | Add policy change tracking, snapshot digests, and telemetry/logging hooks. |
| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-15) | Team Excititor Storage | EXCITITOR-STORAGE-01-001 | Mongo mapping registry plus raw/export entities and DI extensions in place. |
| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-16) | Team Excititor Storage | EXCITITOR-STORAGE-01-004 | Build provider/consensus/cache class maps and related collections. |
| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Export/TASKS.md | DONE (2025-10-15) | Team Excititor Export | EXCITITOR-EXPORT-01-001 | Export engine delivers cache lookup, manifest creation, and policy integration. |
| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Export/TASKS.md | DONE (2025-10-17) | Team Excititor Export | EXCITITOR-EXPORT-01-004 | Connect export engine to attestation client and persist Rekor metadata. |
| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Attestation/TASKS.md | DONE (2025-10-16) | Team Excititor Attestation | EXCITITOR-ATTEST-01-001 | Implement in-toto predicate + DSSE builder providing envelopes for export attestation. |
| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Connectors.Abstractions/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors | EXCITITOR-CONN-ABS-01-001 | Deliver shared connector context/base classes so provider plug-ins can be activated via WebService/Worker. |
| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.WebService/TASKS.md | DONE (2025-10-17) | Team Excititor WebService | EXCITITOR-WEB-01-001 | Scaffold minimal API host, DI, and `/excititor/status` endpoint integrating policy, storage, export, and attestation services. |
| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Worker/TASKS.md | DONE (2025-10-17) | Team Excititor Worker | EXCITITOR-WORKER-01-001 | Create Worker host with provider scheduling and logging to drive recurring pulls/reconciliation. |
| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-CSAF-01-001 | Implement CSAF normalizer foundation translating provider documents into `VexClaim` entries. |
| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CycloneDX/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-CYCLONE-01-001 | Implement CycloneDX VEX normalizer capturing `analysis` state and component references. |
| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.OpenVEX/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-OPENVEX-01-001 | Implement OpenVEX normalizer to ingest attestations into canonical claims with provenance. |
| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors Red Hat | EXCITITOR-CONN-RH-01-001 | Ship Red Hat CSAF provider metadata discovery enabling incremental pulls. |
| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors Red Hat | EXCITITOR-CONN-RH-01-002 | Fetch CSAF windows with ETag handling, resume tokens, quarantine on schema errors, and persist raw docs. |
| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors Red Hat | EXCITITOR-CONN-RH-01-003 | Populate provider trust overrides (cosign issuer, identity regex) and provenance hints for policy evaluation/logging. |
| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors Red Hat | EXCITITOR-CONN-RH-01-004 | Persist resume cursors (last updated timestamp/document hashes) in storage and reload during fetch to avoid duplicates. |
| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors Red Hat | EXCITITOR-CONN-RH-01-005 | Register connector in Worker/WebService DI, add scheduled jobs, and document CLI triggers for Red Hat CSAF pulls. |
| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors Red Hat | EXCITITOR-CONN-RH-01-006 | Add CSAF normalization parity fixtures ensuring RHSA-specific metadata is preserved. |
| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors Cisco | EXCITITOR-CONN-CISCO-01-001 | Implement Cisco CSAF endpoint discovery/auth to unlock paginated pulls. |
| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors Cisco | EXCITITOR-CONN-CISCO-01-002 | Implement Cisco CSAF paginated fetch loop with dedupe and raw persistence support. |
| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors SUSE | EXCITITOR-CONN-SUSE-01-001 | Build Rancher VEX Hub discovery/subscription path with offline snapshot support. |
| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.MSRC.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors MSRC | EXCITITOR-CONN-MS-01-001 | Deliver AAD onboarding/token cache for MSRC CSAF ingestion. |
| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors Oracle | EXCITITOR-CONN-ORACLE-01-001 | Implement Oracle CSAF catalogue discovery with CPU calendar awareness. |
| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors Ubuntu | EXCITITOR-CONN-UBUNTU-01-001 | Implement Ubuntu CSAF discovery and channel selection for USN ingestion. |
| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | TODO | Team Excititor Connectors OCI | EXCITITOR-CONN-OCI-01-001 | Wire OCI discovery/auth to fetch OpenVEX attestations for configured images. |
| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI | EXCITITOR-CLI-01-001 | Add `excititor` CLI verbs bridging to WebService with consistent auth and offline UX. |
| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Core/TASKS.md | TODO | Team Excititor Core & Policy | EXCITITOR-CORE-02-001 | Context signal schema prep extend consensus models with severity/KEV/EPSS fields and update canonical serializers. |
| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Policy/TASKS.md | TODO | Team Excititor Policy | EXCITITOR-POLICY-02-001 | Scoring coefficients & weight ceilings add α/β options, weight boosts, and validation guidance. |
| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | TODO | Team Excititor Storage | EXCITITOR-STORAGE-02-001 | Statement events & scoring signals create immutable VEX statement store plus consensus extensions with indexes/migrations. |
| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Team Excititor WebService | EXCITITOR-WEB-01-004 | Resolve API & signed responses expose `/excititor/resolve`, return signed consensus/score envelopes, document auth. |
| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Attestation/TASKS.md | DONE (2025-10-16) | Team Excititor Attestation | EXCITITOR-ATTEST-01-002 | Rekor v2 client integration ship transparency log client with retries and offline queue. |
| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Worker/TASKS.md | TODO | Team Excititor Worker | EXCITITOR-WORKER-01-004 | TTL refresh & stability damper schedule re-resolve loops and guard against status flapping. |
| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Export/TASKS.md | TODO | Team Excititor Export | EXCITITOR-EXPORT-01-005 | Score & resolve envelope surfaces include signed consensus/score artifacts in exports. |
| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Core/TASKS.md | TODO | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-07-001 | Advisory event log & asOf queries surface immutable statements and replay capability. |
| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Core/TASKS.md | TODO | Team Core Engine & Data Science | FEEDCORE-ENGINE-07-002 | Noise prior computation service learn false-positive priors and expose deterministic summaries. |
| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | TODO | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-07-001 | Advisory statement & conflict collections provision Mongo schema/indexes for event-sourced merge. |
| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Merge/TASKS.md | TODO | BE-Merge | FEEDMERGE-ENGINE-07-001 | Conflict sets & explainers persist conflict materialization and replay hashes for merge decisions. |
| Sprint 8 | Mongo strengthening | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | TODO | Team Normalization & Storage Backbone | FEEDSTORAGE-MONGO-08-001 | Causal-consistent Concelier storage sessions<br>Ensure `AddMongoStorage` registers a scoped session facilitator (causal consistency + majority concerns), update repositories to accept optional session handles, and add integration coverage proving read-your-write and monotonic reads across a replica set/election scenario. |
| Sprint 8 | Mongo strengthening | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Storage Guild | AUTHSTORAGE-MONGO-08-001 | Harden Authority Mongo usage<br>Introduce scoped MongoDB sessions with `writeConcern`/`readConcern` majority defaults, flow the session through stores used in mutations + follow-up reads, and document middleware pattern for web/API & GraphQL layers. |
| Sprint 8 | Mongo strengthening | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | TODO | Team Excititor Storage | EXCITITOR-STORAGE-MONGO-08-001 | Causal consistency for Excititor repositories<br>Register Mongo options with majority defaults, push session-aware overloads through raw/export/consensus/cache stores, and extend migration/tests to validate causal reads after writes (including GridFS-backed content) under replica-set failover. |

2
SPRINTS_EXCITITOR.md Normal file
View File

@@ -0,0 +1,2 @@
| Sprint | Theme | Tasks File Path | Status | Type of Specialist | Task ID | Task Description |
| --- | --- | --- | --- | --- | --- | --- |

View File

@@ -4,7 +4,7 @@
|FEEDCONN-CERTCC-02-005 Deterministic fixtures/tests|DONE (2025-10-11)|Snapshot regression for summary/detail fetch landed; fixtures regenerate via `UPDATE_CERTCC_FIXTURES`.|
|FEEDCONN-CERTCC-02-008 Snapshot coverage handoff|DONE (2025-10-11)|Fixtures + README guidance shipped; QA can rerun with `UPDATE_CERTCC_FIXTURES=1` and share recorded-request diff with Merge.|
|FEEDCONN-CERTCC-02-007 Connector test harness remediation|DONE (2025-10-11)|Harness now resets time provider, wires Source.Common, and verifies VINCE canned responses across fetch→parse→map.|
|FEEDCONN-CERTCC-02-009 Detail/map reintegration plan|DONE (2025-10-11)|Plan published in `src/StellaOps.Feedser.Source.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md`; outlines staged enablement + rollback.|
|FEEDCONN-CERTCC-02-009 Detail/map reintegration plan|DONE (2025-10-11)|Plan published in `src/StellaOps.Concelier.Connector.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md`; outlines staged enablement + rollback.|
# Connector Apple Status
| Task | Status | Notes |

View File

@@ -1,388 +1,430 @@
#7 · HighLevel Architecture — **StellaOps**
Below is the **revised, consolidated** `high_level_architecture.md`.
It **absorbs** all content from `components.md` so you have a single, authoritative file. No separate components doc is required.
---
##0Purpose &Scope
# HighLevel Architecture — **StellaOps** (Consolidated • 2025Q4)
Give contributors, DevOps engineers and auditors a **complete yet readable map** of the Core:
* Major runtime components and message paths.
* Where plugins, CLI helpers and runtime agents attach.
* Technology choices that enable the sub5second SBOM goal.
* Typical operational scenarios (pipeline scan, mute, nightly rescan, etc.).
Anything enterpriseonly (signed PDF, custom/regulated TLS, LDAP, enforcement) **must arrive as a plugin**; the Core never hardcodes those concerns.
---
##1Component Overview
| # | Component | Responsibility |
|---|-----------|---------------|
| 1 | **API Gateway** | REST endpoints (`/scan`, `/quota`, **`/token/offline`**); token auth; quota enforcement |
| 2 | **Scan Service** | SBOM parsing, DeltaSBOM cache, vulnerability lookup |
| 3 | **Policy Engine** | YAML / (optional) Rego rule evaluation; verdict assembly |
| 4 | **Quota Service** | Pertoken counters; **333 scans/day**; waits & HTTP 429 |
| 5 | **ClientJWT Issuer** | Issues 30day offline tokens; bundles them into OUK |
| 5 | **Registry** | Anonymous internal Docker registry for agents, SBOM uploads |
| 6 | **Web UI** | React/Blazor SPA; dashboards, policy editor, quota banner |
| 7 | **Data Stores** | **Redis** (cache, quota) & **MongoDB** (SBOMs, findings, audit) |
| 8 | **Plugin Host** | Hotload .NET DLLs; isolates community plugins |
| 9 | **Agents** | `sbombuilder`, `Stella CLI` scanner CLI, future `StellaOpsAttestor` |
```mermaid
flowchart TD
subgraph "External Actors"
DEV["Developer / DevSecOps / Manager"]
CI["CI/CD Pipeline (e.g., Stella CLI)"]
K8S["Kubernetes Cluster (e.g., Zastava Agent)"]
end
subgraph "Stella Ops Runtime"
subgraph "Core Services"
CORE["Stella Core<br>(REST + gRPC APIs, Orchestration)"]
REDIS[("Redis<br>(Cache, Queues, Trivy DB Mirror)")]
MONGO[("MongoDB<br>(Optional: Long-term Storage)")]
POL["Mute Policies<br>(OPA & YAML Evaluator)"]
REG["StellaOps Registry<br>(Docker Registry v2)"]
ATT["StellaOps Attestor<br>(SLSA + Rekor)"]
end
subgraph "Agents & Builders"
SB["SBOM Builder<br>(Go Binary: Extracts Layers, Generates SBOMs)"]
SA["Stella CLI<br>(Pipeline Helper: Invokes Builder, Triggers Scans)"]
ZA["Zastava Agent<br>(K8s Webhook: Enforces Policies, Inventories Containers)"]
end
subgraph "Scanners & UI"
TRIVY["Trivy Scanner<br>(Plugin Container: Vulnerability Scanning)"]
UI["Web UI<br>(Vue3 + Tailwind: Dashboards, Policy Editor)"]
CLI["Stella CLI<br>(CLI Helper: Triggers Scans, Mutes)"]
end
end
DEV -->|Browses Findings, Mutes CVEs| UI
DEV -->|Triggers Scans| CLI
CI -->|Generates SBOM, Calls /scan| SA
K8S -->|Inventories Containers, Enforces Gates| ZA
UI -- "REST" --> CORE
CLI -- "REST/gRPC" --> CORE
SA -->|Scan Requests| CORE
SB -->|Uploads SBOMs| CORE
ZA -->|Policy Gates| CORE
CORE -- "Queues, Caches" --> REDIS
CORE -- "Persists Data" --> MONGO
CORE -->|Evaluates Policies| POL
CORE -->|Attests Provenance| ATT
CORE -->|Scans Vulnerabilities| TRIVY
SB -- "Pulls Images" --> REG
SA -- "Pulls Images" --> REG
ZA -- "Pulls Images" --> REG
style DEV fill:#f9f,stroke:#333
style CI fill:#f9f,stroke:#333
style K8S fill:#f9f,stroke:#333
style CORE fill:#ddf,stroke:#333
style REDIS fill:#fdd,stroke:#333
style MONGO fill:#fdd,stroke:#333
style POL fill:#dfd,stroke:#333
style REG fill:#dfd,stroke:#333
style ATT fill:#dfd,stroke:#333
style SB fill:#fdf,stroke:#333
style SA fill:#fdf,stroke:#333
style ZA fill:#fdf,stroke:#333
style TRIVY fill:#ffd,stroke:#333
style UI fill:#ffd,stroke:#333
style CLI fill:#ffd,stroke:#333
```
* **Developer / DevSecOps / Manager** browses findings, mutes CVEs, triggers scans.
* **Stella CLI** generates SBOMs and calls `/scan` during CI.
* **Zastava Agent** inventories live containers; Core ships it in *passive* mode only (no kill).
###1.1ClientJWT Lifecycle (offline aware)
1. **Online instance** user signs in → `/connect/token` issues JWT valid 12h.
2. **Offline instance** JWT with `exp 30days` ships in OUK; backend
**resigns** and stores it during import.
3. Tokens embed a `tier` claim (“Free”) and `maxScansPerDay: 333`.
4. On expiry the UI surfaces a red toast **7days** in advance.
> **Purpose.** A complete, implementationready map of StellaOps: product vision, all runtime components, trust boundaries, tokens/licensing, control/data flows, storage, APIs, security, scale, DevOps, and verification logic.
> **Scope.** This file **replaces** the separate `components.md`; all component details now live here.
---
##2·Component Responsibilities (runtime view)
## 0) Product vision & principles
| Component | Core Responsibility | Implementation Highlights |
| -------------------------- | ---------------------------------------------------------------------------------------------------------- | --------------------------------------------------------- |
| **Stella Core** | Orchestrates scans, persists SBOM blobs, serves REST/gRPC APIs, fans out jobs to scanners & policy engine. | .NET{{ dotnet }}, CQRS, Redis Streams; pluggable runner interfaces. |
| **SBOM Builder** | Extracts image layers, queries Core for *missing* layers, generates SBOMs (multiformat), uploads blobs. | Go binary; wraps Trivy & Syft libs. |
| **Stella CLI** | Pipelineside helper; invokes Builder, triggers scan, streams progress back to CI/CD. | Static musl build. |
| **Zastava Agent** | K8s admission webhook enforcing policy verdicts before Pod creation. | Rust for sub10ms latencies. |
| **UI** | Angular17 SPA for dashboards, settings, policy editor. | Tailwind CSS; Webpack module federation (future). |
| **Redis** | Cache, queue, TrivyDB mirror, layer diffing. | Single instance or Sentinel. |
| **MongoDB** (opt.) | Longterm SBOM & policy audit storage (>180days). | Optional; enabled via flag. |
| **StellaOps.Registry** | Anonymous readonly Docker v2 registry with optional Cosign verification. | `registry :2` behind nginx reverse proxy. |
| **StellaOps.MutePolicies** | YAML/Rego evaluator, policy version store, `/policy/*` API. | Embeds OPAWASM; falls back to `opa exec`. |
| **StellaOpsAttestor** | Generate SLSA provenance & Rekor signatures; verify on demand. | Sidecar container; DSSE + Rekor CLI. |
**Vision.** StellaOps is a **deterministic SBOM + VEX platform** for CI/CD and runtime, tuned for **speed** (perlayer deltas), **quiet output** (usagescoped views), and **verifiability** (DSSE + Rekor v2). It is **selfhostable**, **airgap capable**, and **commercially enforceable**: only licensed installations can produce **StellaOpsverified** attestations.
All crosscomponent calls use dependencyinjected interfaces—no
intracomponent reachins.
**Operating principles.**
* **Scannerowned SBOMs.** We generate our own BOMs; we do not warehouse thirdparty SBOM content (we can **link** to attested SBOMs).
* **Deterministic evidence.** Facts come from package DBs, installed metadata, linkers, and verified attestations; no fuzzy guessing in the core.
* **Perlayer caching.** Cache fragments by **layer digest** and compose image SBOMs via **CycloneDX BOMLink** / **SPDX ExternalRef**.
* **Inventory vs Usage.** Always record the full **inventory** of what exists; separately present **usage** (entrypoint closure + loaded libs).
* **Backend decides.** PASS/FAIL is produced by **Policy** + **VEX** + **Advisories**. The scanner reports facts.
* **Attest or it didnt happen.** Every export is signed as **intoto/DSSE** and logged in **Rekor v2**.
* **Sovereignready.** Cloud is used only for licensing and optional endorsement; everything else is firstparty and selfhostable.
---
##3·Principal Backend Modules & Plugin Hooks
## 1) Service topology & trust boundaries
| Namespace | Responsibility | Builtin Tech / Default | Plugin Contract |
| --------------- | -------------------------------------------------- | ----------------------- | ------------------------------------------------- |
| `configuration` | Parse env/JSON, healthcheck endpoint | .NET{{ dotnet }} Options | `IConfigValidator` |
| `identity` | Embedded OAuth2/OIDC (OpenIddict 6) | MIT OpenIddict | `IIdentityProvider` for LDAP/SAML/JWT gateway |
| `pluginloader` | Discover DLLs, SemVer gate, optional Cosign verify | Reflection + Cosign | `IPluginLifecycleHook` for telemetry |
| `scanning` | SBOM & imageflow orchestration; runner pool | Trivy CLI (default) | `IScannerRunner` e.g., Grype, Copacetic, Clair |
| `feedser` (vulnerability ingest/merge/export service) | Nightly NVD merge & feed enrichment | Hangfire job | drop-in `*.Schedule.dll` for OSV, GHSA, NVD 2.0, CNNVD, CNVD, ENISA, JVN and BDU feeds |
| `tls` | TLS provider abstraction | OpenSSL | `ITlsProvider` for custom suites (incl. **SM2**, where law or security requires it) |
| `reporting` | Render HTML/PDF reports | RazorLight | `IReportRenderer` |
| `ui` | Angular SPA & i18n | Angular{{ angular }} | new locales via `/locales/{lang}.json` |
| `scheduling` | Cron + retries | Hangfire | any recurrent job via `*.Schedule.dll` |
### 1.1 Runtime inventory (firstparty)
```mermaid
classDiagram
class configuration
class identity
class pluginloader
class scanning
class feedser
class tls
class reporting
class ui
class scheduling
| Service / Tool | Container image | Core role | Scale pattern |
| ------------------------------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------- |
| **Scanner.WebService** | `stellaops/scanner-web` | Control plane for scans; catalog; SBOM composition (inventory & usage); diff; exports. | Stateless; N replicas behind LB. |
| **Scanner.Worker** | `stellaops/scanner-worker` | Runs analyzers (OS, Lang: Java/Node/Python/Go/.NET/Rust, Native ELF/PE/MachO, EntryTrace); emits perlayer SBOMs and composes image SBOMs. | Horizontal; queuedriven; sharded by layer digest. |
| **Scanner.Sbomer.BuildXPlugin** | `stellaops/sbom-indexer` | BuildKit **generator** for buildtime SBOMs as OCI **referrers**. | CIside; ephemeral. |
| **Scanner.Sbomer.DockerImage** | `stellaops/scanner-cli` | CLIorchestrated scanner container for postbuild scans. | Local/CI; ephemeral. |
| **Concelier.WebService** | `stellaops/concelier-web` | Vulnerability ingest/normalize/merge/export (JSON + Trivy DB). | HA via Mongo locks. |
| **Excititor.WebService** | `stellaops/excititor-web` | VEX ingest/normalize/consensus; conflict retention; exports. | HA via Mongo locks. |
| **Policy Engine** | (in `scanner-web`) | YAML DSL evaluator (waivers, vendor preferences, KEV/EPSS, license, usagegating); produces **policy digest**. | Inprocess; cache per digest. |
| **Signer** | `stellaops/signer` | **Hard gate:** validates entitlement + release integrity; mints signing cert (Fulcio keyless) or uses KMS; signs DSSE. | Stateless; HPA by QPS. |
| **Attestor** | `stellaops/attestor` | Posts DSSE bundles to **Rekor v2**; verification endpoints. | Stateless; HPA by QPS. |
| **Authority** | `stellaops/authority` | Onprem OIDC issuing **shortlived OpToks** with DPoP/mTLS sender constraint. | HA behind LB. |
| **Zastava** (Runtime) | `stellaops/zastava` | Runtime inspector/enforcer (observer + optional Admission Webhook). | DaemonSet + Webhook. |
| **Web UI** | `stellaops/ui` | Angular app for scans, diffs, policy, VEX, runtime, reports. | Stateless. |
| **StellaOps.Cli** | `stellaops/cli` | CLI for init/scan/export/diff/policy/report/verify; Buildx helper. | Local/CI. |
class AllModules
### 1.2 Thirdparty (selfhosted)
configuration ..> identity : Uses
identity ..> pluginloader : Authenticates Plugins
pluginloader ..> scanning : Loads Scanner Runners
scanning ..> feedser : Triggers Feed Merges
tls ..> AllModules : Provides TLS Abstraction
reporting ..> ui : Renders Reports for UI
scheduling ..> feedser : Schedules Nightly Jobs
* **Fulcio** (Sigstore CA) — issues shortlived signing certs (keyless).
* **Rekor v2** (tilebacked transparency log).
* **MinIO** — S3compatible object store with lifecycle & Object Lock.
* **MongoDB** — catalog, advisories, VEX.
* **Queue** — Redis Streams / NATS / RabbitMQ (pluggable).
* **OCI Registry** — must support **Referrers API** (discover SBOMs/signatures).
note for scanning "Pluggable: ISScannerRunner<br>e.g., Trivy, Grype"
note for feedser "Pluggable: *.Schedule.dll<br>e.g., OSV, GHSA Feeds"
note for identity "Pluggable: IIdentityProvider<br>e.g., LDAP, SAML"
note for reporting "Pluggable: IReportRenderer<br>e.g., Custom PDF"
```
### 1.3 Cloud licensing (StellaOps)
**When remaining =0:**
API returns `429 Too Many Requests`, `RetryAfter: <UTCmidnight>` (sequence omitted for brevity).
* **Licensing Service** (`www.stella-ops.org`) — issues longlived **License Tokens (LT)**; exchanges LT → **ProofofEntitlement (PoE)** bound to an installation key; revoke/introspect PoE; optional crosslog **endorsement**.
---
##4·Data Flows
###4.1SBOMFirst (≤5s P95)
Builder produces SBOM locally, so Core never touches the Docker
socket.
Trivy path hits ≤5s on alpine:3.19 with warmed DB.
Imageunpack fallback stays ≤10s for 200MB images.
```mermaid
sequenceDiagram
participant CI as CI/CD Pipeline (Stella CLI)
participant SB as SBOM Builder
participant CORE as Stella Core
participant REDIS as Redis Queue
participant RUN as Scanner Runner (e.g., Trivy)
participant POL as Policy Evaluator
CI->>SB: Invoke SBOM Generation
SB->>CORE: Check Missing Layers (/layers/missing)
CORE->>REDIS: Query Layer Diff (SDIFF)
REDIS-->>CORE: Missing Layers List
CORE-->>SB: Return Missing Layers
SB->>SB: Generate Delta SBOM
SB->>CORE: Upload SBOM Blob (POST /scan(sbom))
CORE->>REDIS: Enqueue Scan Job
REDIS->>RUN: Fan Out to Runner
RUN->>RUN: Perform Vulnerability Scan
RUN-->>CORE: Return Scan Results
CORE->>POL: Evaluate Mute Policies
POL-->>CORE: Policy Verdict
CORE-->>CI: JSON Verdict & Progress Stream
Note over CORE,CI: Achieves ≤5s P95 with Warmed DB
```
###4.2Delta SBOM
Builder collects layer digests.
`POST /layers/missing` → Redis SDIFF → missing layer list (<20ms).
SBOM generated only for those layers and uploaded.
###4.3Feedser Harvest & Export
```mermaid
sequenceDiagram
participant SCHED as Feedser Scheduler
participant CONN as Source Connector Plug-in
participant FEEDSER as Feedser Core
participant MONGO as MongoDB (Canonical Advisories)
participant EXPORT as Exporter (JSON / Trivy DB)
participant ART as Artifact Store / Offline Kit
SCHED->>CONN: Trigger window (init/resume)
CONN->>CONN: Fetch source documents + metadata
CONN->>FEEDSER: Submit raw document for parsing
FEEDSER->>FEEDSER: Parse & normalize to DTO
FEEDSER->>FEEDSER: Merge & deduplicate canonical advisory
FEEDSER->>MONGO: Write advisory, provenance, merge_event
FEEDSER->>EXPORT: Queue export delta request
EXPORT->>MONGO: Read canonical snapshot/deltas
EXPORT->>EXPORT: Build deterministic JSON & Trivy DB artifacts
EXPORT->>ART: Publish artifacts / Offline Kit bundle
ART-->>FEEDSER: Record export state + digests
```
###4.4Identity & Auth Flow
OpenIddict issues JWTs via clientcredentials or password grant.
An IIdentityProvider plugin can delegate to LDAP, SAML or external OIDC
without Core changes.
---
##5·Runtime Helpers
| Helper | Form | Purpose | Extensible Bits |
|-----------|---------------------------------------|--------------------------------------------------------------------|-------------------------------------------|
| **Stella CLI** | Distroless CLI | Generates SBOM, calls `/scan`, honours threshold flag | `--engine`, `--pdf-out` piped to plugins |
| **Zastava** | Static Go binary / DaemonSet | Watches Docker/CRIO events; uploads SBOMs; can enforce gate | Policy plugin could alter thresholds |
---
##6·Persistence & Cache Strategy
| Store | Primary Use | Why chosen |
|----------------|-----------------------------------------------|--------------------------------|
| **MongoDB** | Feedser canonical advisories, merge events, export state | Deterministic canonical store with flexible schema |
| **Redis7** | CLI quotas, short-lived job scheduling, layer diff cache | Sub-1ms P99 latency for hot-path coordination |
| **Local tmpfs**| Trivy layer cache (`/var/cache/trivy`) | Keeps disk I/O off hot path |
### 1.4 Diagram (control/data planes & trust)
```mermaid
flowchart LR
subgraph "Persistence Layers"
REDIS[(Redis: Quotas & Short-lived Queues<br>Sub-1ms P99)]
MONGO[(MongoDB: Canonical Advisories<br>Merge Events & Export State)]
TMPFS[(Local tmpfs: Trivy Layer Cache<br>Low I/O Overhead)]
end
subgraph Cloud["www.stella-ops.org (Cloud)"]
LS[Licensing Service<br/>LT→PoE / revoke / introspect]
end
CORE["Stella Core"] -- Queues & SBOM Cache --> REDIS
CORE -- Long-term Storage --> MONGO
TRIVY["Trivy Scanner"] -- Layer Unpack Cache --> TMPFS
subgraph OnPrem["Customer Site (Self-hosted)"]
Auth[Authority (OIDC)\nOpTok (DPoP/mTLS)]
SW[Scanner.WebService]
WK[Scanner.Worker xN]
FEED[Concelier]
VEX[Excititor]
POL[Policy Engine (in Scanner.Web)]
SGN[Signer\n(entitlement + signing)]
ATT[Attestor\n(Rekor v2 submit/verify)]
UI[Web UI (Angular)]
Z[Zastava\n(Runtime Inspector/Enforcer)]
MIN[(MinIO S3)]
MGO[(MongoDB)]
QUE[(Queue/Streams)]
end
style REDIS fill:#fdd,stroke:#333
style MONGO fill:#dfd,stroke:#333
style TMPFS fill:#ffd,stroke:#333
CLI[StellaOps.Cli / Buildx Plugin]
REG[(OCI Registry with Referrers)]
FUL[ Fulcio ]
REK[ Rekor v2 (tiles) ]
CLI -->|scan/build| SW
SW -->|jobs| QUE
QUE --> WK
WK --> MIN
SW --> MGO
FEED --> MGO
VEX --> MGO
UI --> SW
Z --> SW
SGN <--> Auth
SGN --> FUL
SGN -->|mTLS| ATT
ATT --> REK
SGN <-->|verify referrers| REG
```
**Trust boundaries.** Only **Signer** can sign; only **Attestor** can write to **Rekor v2**. Scanner/UI never sign.
---
## 2) Licensing & tokens (installationready, theftresistant)
**Twotoken model.**
* **License Token (LT)** — longlived JWT from **Licensing Service**; used **once** to enroll the installation; never used in hot path.
* **ProofofEntitlement (PoE)** — bound to the installation key (mTLS client cert **or** DPoPbound JWT with `cnf`); mediumlived; renewable; revocable.
* **Operational token (OpTok)** — 25min OIDC token from **Authority**, **senderconstrained** (DPoP or mTLS). Used to authenticate to **Signer**/**Scanner.WebService**.
**Signer enforces both:** PoE proves entitlement; OpTok proves “who is calling now”. It also **independently verifies** the **scanner image digest** is **StellaOpssigned** via **Referrers + cosign** before signing anything.
**Enrollment sequence (LT → PoE).**
```plantuml
@startuml
actor Operator
participant "Install Agent" as IA
participant "Licensing Service" as LS
Operator -> IA: Provide LT
IA -> IA: Generate K_inst
IA -> LS: /license/enroll {LT, pub(K_inst)}
LS --> IA: PoE (mTLS client cert or JWT with cnf=K_inst), CRL/OCSP/introspect
@enduml
```
---
##7·Typical Scenarios
## 3) Scanner subsystem (facts engine)
| # | Flow | Steps |
|---------|----------------------------|-------------------------------------------------------------------------------------------------|
| **S1** | Pipeline Scan & Alert | Stella CLI SBOM `/scan` policy verdict CI exit code & link to *Scan Detail* |
| **S2** | Mute Noisy CVE | Dev toggles **Mute** in UI rule stored in Redis next build passes |
| **S3** | Nightly Rescan | `SbomNightly.Schedule` requeues SBOMs (maskfilter) dashboard highlights new Criticals |
| **S4** | Feed Update Cycle | `Feedser (vulnerability ingest/merge/export service)` refreshes feeds UI *Feed Age* tile turns green |
| **S5** | Custom Report Generation | Plugin registers `IReportRenderer` `/report/custom/{digest}` CI downloads artifact |
### 3.1 Analyzers (deterministic only)
* **OS packages:** apk/dpkg/rpm (Linux); Windows MSI/SxS/GAC (M2).
* **Language (installed state):**
* Java (pom.properties / MANIFEST) → `pkg:maven/...`
* Node (`node_modules/*/package.json`) → `pkg:npm/...`
* Python (`*.dist-info/METADATA`) → `pkg:pypi/...`
* Go (buildinfo) → `pkg:golang/...`
* .NET (`*.deps.json`) → `pkg:nuget/...`
* **Rust:** deterministic **language markers** (symbol mangling) and crates only when present; otherwise `bin:{sha256}`.
* **Native:** ELF/PE/MachO imports, DT_NEEDED, RPATH/RUNPATH, symbol versions, PE version info.
* **EntryTrace:** parse `ENTRYPOINT`/`CMD`; shell AST; resolve launchers (Java/Node/Python) to terminal program; record file:line chain.
### 3.2 Caching & composition
* **Layer cache:** `{layerDigest → SBOM fragment + analyzer meta}`.
* **File CAS:** `{sha256(file) → parse result (ELF/JAR metadata/etc.)}`.
* **Composition:** build **image SBOMs** from fragments via **BOMLink/ExternalRef**; emit **two views**:
* **Inventory** (complete filesystem inventory).
* **Usage** (entrypoint closure + linked libs).
* **Transport:** JSON **and** **CycloneDX Protobuf** (compact, fast to parse).
* **Index:** BOMIndex sidecar with purl table + roaring bitmap + `usedByEntrypoint` flag for fast joins.
### 3.3 Diff (image → layer → package)
* Added / Removed / Versionchanged changes, **attributed** to the layer that caused them.
* Raw diffs preserved; backend view applies **VEX + Policy**.
### 3.4 Buildtime SBOMs (fast CI path)
* Buildx **generator** runs analyzers during `docker buildx build --attest=type=sbom,generator=stellaops/sbom-indexer`, attaches SBOMs as **OCI referrers**.
* Scanner.WebService can trust these (policyconfigurable) and **skip** rescan; DSSE + Rekor v2 can be done either at build time or postpush via Signer/Attestor.
---
## 4) Backend evaluation (decider)
### 4.1 Concelier (advisories)
* Ingests vendor, distro, OSS feeds; normalizes & merges; persists canonical advisories in Mongo; exports **deterministic JSON** and **Trivy DB**.
* Offline kit bundles for airgapped sites.
### 4.2 Excititor (VEX)
* Ingests **OpenVEX / CSAF VEX / CycloneDX VEX**; normalizes claims; retains conflicts; computes **consensus** with provider trust weights and justification gates.
### 4.3 Policy Engine (YAML DSL)
* Matchers: `image/repo/env/purl/cve/vendor/source/path/layerDigest/usedByEntrypoint`
* Actions: `ignore(until, justification)`, `fail`, `warn`, `defer`, `requireVEX{vendors, justifications}`, `escalate {sev, KEV, EPSS}`, license constraints.
* Produces a **policy digest** (SHA256 of canonicalized policy).
### 4.4 PASS/FAIL flow
1. SBOM (Inventory / Usage) → join with **Concelier** advisories.
2. Apply **Excititor** consensus (statuses & justifications).
3. Apply **Policy**; compute PASS/FAIL with waiver TTLs.
4. Sign the **final report** (DSSE via **Signer**) and log to **Rekor v2** via **Attestor**.
---
## 5) Runtime enforcement (Zastava)
* **Observer:** inventories running containers, checks image signatures, SBOM presence (referrers), detects drift (entrypoint chain divergence), flags unapproved images.
* **Admission Webhook (optional):** blocks policyfail pods (dryrun first).
* **Integration:** posts runtime events to Scanner.WebService; can request **delta scans** on changed layers.
---
## 6) Storage & catalogs (MinIO/Mongo)
**MinIO layout**
```
s3://stellaops/
layers/<sha256>/sbom.cdx.json.zst
layers/<sha256>/sbom.spdx.json.zst
images/<imgDigest>/inventory.cdx.pb
images/<imgDigest>/usage.cdx.pb
indexes/<imgDigest>/bom-index.bin
attest/<artifactSha256>.dsse.json
```
**Catalog (Mongo)**
* `artifacts` (type/format/sha/size/rekor/ttl/immutable/refCount/createdAt)
* `images`, `layers`, `links`, `lifecycleRules`
**Retention**
* MinIO **ILM** for coarse TTL; Scanner.WebService GC decrements `refCount` and deletes unreferenced metadata; **Object Lock** for immutable classes (auditable artifacts).
---
## 7) APIs (consolidated surface)
### 7.1 Scanner.WebService
```
POST /api/scans { imageRef|digest, force? } → { scanId }
GET /api/scans/{id} → { status, digests, artifacts[] }
GET /api/sboms/{imageDigest} ?format=cdx-json|cdx-pb|spdx-json&view=inventory|usage
GET /api/diff?old=<digest>&new=<digest> → { added[], removed[], changed[], byLayer[] }
POST /api/exports { imageDigest, format, view } → { artifactId, rekorUrl }
POST /api/reports { imageDigest, policyRevision? } → { reportId, rekorUrl }
GET /api/catalog/artifacts/{id} → { size, ttl, immutable, rekor, refs }
GET /healthz | /readyz | /metrics
```
### 7.2 Signer (mTLS; hard gate)
```
POST /sign/dsse # body: {subjectHash, imageDigest, predicate}; headers: OpTok (DPoP/mTLS) + PoE
GET /verify/referrers?imageDigest=sha256:... # is this image StellaOps-signed?
```
### 7.3 Attestor (mTLS)
```
POST /rekor/entries # DSSE bundle → {uuid, index, proof, logURL}
GET /rekor/entries/{uuid}
```
### 7.4 Authority (OIDC)
* `/.well-known/openid-configuration`, `/oauth/token` (DPoP/mTLS), `/oauth/introspect`, `/jwks`
### 7.5 Licensing (cloud)
```
POST /license/enroll { LT, pubKey } → PoE + introspection endpoints
POST /license/revoke { license_id } → ok
POST /license/introspect { poe } → { active, claims, exp }
POST /attest/endorse { bundle } → endorsement bundle (optional)
```
---
## 8) Security & verifiability
* **Senderconstrained tokens.** All operational calls use **DPoP** (RFC9449) or **mTLSbound** tokens (RFC8705).
* **Entitlement.** **PoE** is mandatory; revocation honored online.
* **Release integrity.** **Signer** independently verifies **scanner image digest** via **Referrers + cosign** before signing.
* **Separation of duties.** Scanner/UI cannot sign; only **Signer** can sign; only **Attestor** can write to **Rekor v2**.
* **Verifiers.** Anyone can verify: DSSE signature → certificate chain to **StellaOps Fulcio/KMS root****Rekor v2** inclusion.
* **Community vs Authorized.** Free/community runs throttled with no official attestations; authorized runs full speed and produce **StellaOpsverified** bundles.
**DSSE predicate (SBOM/report)**
```json
{
"predicateType": "https://stella-ops.org/attestations/sbom/1",
"subject": [{ "name": "s3://stellaops/images/<digest>/inventory.cdx.pb", "digest": { "sha256": "<sha256>" } }],
"predicate": {
"image_digest": "<sha256:...>",
"stellaops_version": "2.3.1 (2027.04)",
"license_id": "LIC-9F2A...",
"customer_id": "CUST-ACME",
"plan": "pro",
"policy_digest": "sha256:...",
"views": ["inventory","usage"],
"created": "2025-10-17T12:34:56Z"
}
}
```
**BOMIndex sidecar**
Binary header + purl table + roaring bitmaps; optional `usedByEntrypoint` flags for fast policy joins.
---
## 9) Scale, performance & quotas
* **Workers:** horizontal; **distributed lock per layer digest**; global CAS in MinIO.
* **Queues:** Redis Streams / NATS / RabbitMQ. HPA by queue depth, CPU, memory.
* **Registry throttling:** perregistry concurrency budgets.
* **Targets:**
* Buildtime path P95 ≤35s on warmed bases.
* Postbuild delta scan P95 ≤10s for 200MB images.
* Policy + VEX evaluation ≤500ms for 5k components using BOMIndex.
* **Quotas:** license plan enforces QPS/concurrency/size; **Signer** throttles and can deny DSSE.
---
## 10) DevOps & distribution
* **Releases:** all firstparty images **cosignsigned**; labels embed `org.stellaops.version` and `org.stellaops.release_date`.
* **Channels:**
* **Community** (public registry): throttled, nonattesting.
* **Authorized** (private registry): full speed, DSSE enabled.
* **Client update flow:** containers selfverify signatures at boot; report version; **Signer** enforces `valid_release_year` / `max_version` from PoE before signing.
* **Compose skeleton:**
```yaml
services:
authority: { image: stellaops/authority }
fulcio: { image: sigstore/fulcio }
rekor: { image: sigstore/rekor-v2 }
minio: { image: minio/minio, command: server /data --console-address ":9001" }
mongo: { image: mongo:7 }
signer: { image: stellaops/signer, depends_on: [authority, fulcio] }
attestor: { image: stellaops/attestor, depends_on: [rekor, signer] }
scanner-web:{ image: stellaops/scanner-web, depends_on: [mongo, minio, signer, attestor] }
scanner-worker:
image: stellaops/scanner-worker
deploy: { replicas: 4 }
depends_on: [scanner-web]
concelier: { image: stellaops/concelier-web, depends_on: [mongo] }
excititor: { image: stellaops/excititor-web, depends_on: [mongo] }
ui: { image: stellaops/ui, depends_on: [scanner-web, concelier, excititor] }
```
* **Backups:** Mongo dumps; MinIO versioned buckets & replication; Rekor v2 DB snapshots; JWKS/Fulcio/KMS key rotation.
---
## 11) Observability & audit
* **Metrics:** scan latency, layer cache hit %, artifact bytes, DSSE/Rekor latency, policy evaluation time, queue depth, admission decisions (Zastava).
* **Tracing:** perstage spans; correlation IDs across Scanner→Signer→Attestor.
* **Audit logs:** every signing records `license_id`, `image_digest`, `policy_digest`, and Rekor UUID.
* **Compliance:** MinIO **Object Lock** for immutable artifacts; reproducible outputs via policy digest + SBOM digest in predicate.
---
## 12) Roadmap (anchored to this architecture)
* M2: Windows MSI/SxS/GAC analyzers; deeper Rust (DWARF enrichers).
* M2: Buildx generator certified flows; crossregistry trust policies.
* M3: PatchPresence plugin (signaturebased backport detection), optin.
* M3: Zastava Admission control GA with policy presets and dryrun→enforce stages.
* Continuous: Policy UX (waiver TTLs, vendor rules), Excititor connectors expansion.
---
## 13) Canonical sequences (verification & signing)
**Sign & log (OpTok + PoE, image verify, DSSE, Rekor).**
```mermaid
sequenceDiagram
participant DEV as Developer
participant UI as Web UI
participant CORE as Stella Core
participant REDIS as Redis
participant RUN as Scanner Runner
autonumber
participant Scan as Scanner.WebService
participant Auth as Authority (OIDC)
participant Sign as Signer
participant Reg as OCI Registry
participant Ful as Fulcio/KMS
participant Att as Attestor
participant Rek as Rekor v2
DEV->>UI: Toggle Mute for CVE
UI->>CORE: Update Mute Rule (POST /policy/mute)
CORE->>REDIS: Store Mute Policy
Note over CORE,REDIS: YAML/Rego Evaluator Updates
alt Next Pipeline Build
CI->>CORE: Trigger Scan (POST /scan)
CORE->>RUN: Enqueue & Scan
RUN-->>CORE: Raw Findings
CORE->>REDIS: Apply Mute Policies
REDIS-->>CORE: Filtered Verdict (Passes)
CORE-->>CI: Success Exit Code
end
Scan->>Auth: Get OpTok (DPoP/mTLS)
Scan->>Sign: sign(request) + OpTok + PoE + DPoP proof
Sign->>Auth: Validate OpTok & sender-constraint
Sign->>Sign: Validate PoE (introspect/revocation)
Sign->>Reg: Verify scanner image is StellaOps-signed (Referrers + cosign)
alt OK
Sign->>Ful: Get signing cert (keyless) or use KMS key
Sign-->>Scan: DSSE bundle (cert chain)
Scan->>Att: Submit bundle
Att-->>Rek: Create entry
Rek-->>Att: {uuid,index,proof}
Att-->>Scan: Rekor URL
else Deny
Sign-->>Scan: 403 (no attestation)
end
```
```mermaid
sequenceDiagram
participant CRON as SbomNightly.Schedule
participant CORE as Stella Core
participant REDIS as Redis Queue
participant RUN as Scanner Runner
participant UI as Dashboard
**Verification (third party).**
CRON->>CORE: Re-queue SBOMs (Mask-Filter)
CORE->>REDIS: Enqueue Filtered Jobs
REDIS->>RUN: Fan Out to Runners
RUN-->>CORE: New Scan Results
CORE->>UI: Highlight New Criticals
Note over CORE,UI: Focus on Changes Since Last Scan
```plantuml
@startuml
actor Verifier
participant "stellaops verify" as Tool
database "Fulcio/KMS root" as Root
participant "Rekor v2" as R2
Verifier -> Tool: bundle (URL/file)
Tool -> Tool: Verify DSSE signature
Tool -> Root: Verify cert chain to StellaOps root
Tool -> R2: Verify inclusion proof / query by UUID
Tool -> Verifier: OK + claims (license_id, policy_digest, version)
@enduml
```
---
##8·UIFastFacts
* **Stack** Angular17 + Vite dev server; Tailwind CSS.
* **State** Signals + RxJS for live scan progress.
* **i18n / l10n** JSON bundles served from `/locales/{lang}.json`.
* **ModuleStructure** Lazyloaded feature modules (`dashboard`, `scans`, `settings`); runtime route injection by UI plugins (roadmap Q22026).
---
##9·CrossCutting Concerns
* **Security** containers run nonroot, `CAP_DROP:ALL`, readonly FS, hardened seccomp profiles.
* **Observability** Serilog JSON, OpenTelemetry OTLP exporter, Prometheus `/metrics`.
* **Upgrade Policy** `/api/v1` endpoints & CLI flags stable across a minor; breaking changes bump major.
---
##10·Performance & Scalability
| Scenario | P95 target | Bottleneck | Mitigation |
|-----------------|-----------:|-----------------|-------------------------------------------------|
| SBOMfirst | 5s | Redis queue | More CPU, increase `ScannerPool.Workers` |
| Imageunpack | 10s | Layer unpack | Prefer SBOM path, warm Docker cache |
| High concurrency| 40rps | Runner CPU | Scale Core replicas + sidecar scanner services |
---
##11·Future Architectural Anchors
* **ScanService microsplit (gRPC)** isolate heavy runners for large clusters.
* **UI route plugins** dynamic Angular module loader (roadmap Q22026).
* **Redis Cluster** transparently sharded cache once sustained>100rps.
---
##12·Assumptions & Tradeoffs
Requires Docker/CRIO runtime; .NET9 available on hosts; Windows containers are outofscope this cycle.
Embedded auth simplifies deployment but may need plugins for enterprise IdPs.
Speed is prioritised over exhaustive feature parity with heavyweight commercial scanners.
---
##13·References & Further Reading
* **C4 Model** <https://c4model.com>
* **.NET Architecture Guides** <https://learn.microsoft.com/dotnet/architecture>
* **OSS Examples** Kubernetes Architecture docs, Prometheus design papers, Backstage.
*(End of HighLevel Architecture v2.2)*
**End of `high_level_architecture.md` (Consolidated).**

View File

@@ -1,208 +0,0 @@
#8 · Detailed Module Specifications — **StellaOps Feedser**
_This document describes the Feedser service, its supporting libraries, connectors, exporters, and test assets that live in the OSS repository._
---
##0Scope
Feedser is the vulnerability ingest/merge/export subsystem of StellaOps. It
fetches primary advisories, normalizes and deduplicates them into MongoDB, and
produces deterministic JSON and Trivy DB exports. This document lists the
projects that make up that workflow, the extension points they expose, and the
artefacts they ship.
---
##1Repository layout (current)
```text
src/
├─ Directory.Build.props / Directory.Build.targets
├─ StellaOps.Plugin/
├─ StellaOps.Feedser.Core/
├─ StellaOps.Feedser.Core.Tests/
├─ StellaOps.Feedser.Models/ (+ .Tests/)
├─ StellaOps.Feedser.Normalization/ (+ .Tests/)
├─ StellaOps.Feedser.Merge/ (+ .Tests/)
├─ StellaOps.Feedser.Storage.Mongo/ (+ .Tests/)
├─ StellaOps.Feedser.Exporter.Json/ (+ .Tests/)
├─ StellaOps.Feedser.Exporter.TrivyDb/ (+ .Tests/)
├─ StellaOps.Feedser.Source.* / StellaOps.Feedser.Source.*.Tests/
├─ StellaOps.Feedser.Testing/
├─ StellaOps.Feedser.Tests.Shared/
├─ StellaOps.Feedser.WebService/ (+ .Tests/)
├─ PluginBinaries/
└─ StellaOps.Feedser.sln
```
Each folder is a .NET project (or set of projects) referenced by
`StellaOps.Feedser.sln`. Build assets are shared through the root
`Directory.Build.props/targets` so conventions stay consistent.
---
##2Shared libraries
| Project | Purpose | Key extension points |
|---------|---------|----------------------|
| `StellaOps.Plugin` | Base contracts for connectors, exporters, and DI routines plus Cosign validation helpers. | `IFeedConnector`, `IExporterPlugin`, `IDependencyInjectionRoutine` |
| `StellaOps.DependencyInjection` | Composable service registrations for Feedser and plug-ins. | `IDependencyInjectionRoutine` discovery |
| `StellaOps.Feedser.Testing` | Common fixtures, builders, and harnesses for integration/unit tests. | `FeedserMongoFixture`, test builders |
| `StellaOps.Feedser.Tests.Shared` | Shared assembly metadata and fixtures wired in via `Directory.Build.props`. | Test assembly references |
---
##3Core projects
| Project | Responsibility | Extensibility |
|---------|----------------|---------------|
| `StellaOps.Feedser.WebService` | ASP.NET Core minimal API hosting Feedser jobs, status endpoints, and scheduler. | DI-based plug-in discovery; configuration binding |
| `StellaOps.Feedser.Core` | Job orchestration, connector pipelines, merge workflows, export coordination. | `IFeedConnector`, `IExportJob`, deterministic merge policies |
| `StellaOps.Feedser.Models` | Canonical advisory DTOs and enums persisted in MongoDB and exported artefacts. | Partial classes for source-specific metadata |
| `StellaOps.Feedser.Normalization` | Version comparison, CVSS normalization, text utilities for canonicalization. | Helpers consumed by connectors/merge |
| `StellaOps.Feedser.Merge` | Precedence evaluation, alias graph maintenance, merge-event hashing. | Policy extensions via DI |
| `StellaOps.Feedser.Storage.Mongo` | Repository layer for documents, DTOs, advisories, merge events, export state. | Connection string/config via options |
| `StellaOps.Feedser.Exporter.Json` | Deterministic vuln-list JSON export pipeline. | Dependency injection for storage + plugin to host |
| `StellaOps.Feedser.Exporter.TrivyDb` | Builds Trivy DB artefacts from canonical advisories. | Optional ORAS push routines |
###3.1StellaOps.Feedser.WebService
* Hosts minimal API endpoints (`/health`, `/status`, `/jobs`).
* Runs the scheduler that triggers connectors and exporters according to
configured windows.
* Applies dependency-injection routines from `PluginBinaries/` at startup only
(restart-time plug-ins).
###3.2StellaOps.Feedser.Core
* Defines job primitives (fetch, parse, map, merge, export) used by connectors.
* Coordinates deterministic merge flows and writes `merge_event` documents.
* Provides telemetry/log scopes consumed by WebService and exporters.
###3.3StellaOps.Feedser.Storage.Mongo
* Persists raw documents, DTO records, canonical advisories, aliases, affected
packages, references, merge events, export state, and job leases.
* Exposes repository helpers for exporters to stream full/delta snapshots.
###3.4StellaOps.Feedser.Exporter.*
* `Exporter.Json` mirrors the Aqua vuln-list tree with canonical ordering.
* `Exporter.TrivyDb` builds Trivy DB Bolt archives and optional OCI bundles.
* Both exporters honour deterministic hashing and respect export cursors.
---
##4Source connectors
Connectors live under `StellaOps.Feedser.Source.*` and conform to the interfaces
in `StellaOps.Plugin`.
| Family | Project(s) | Notes |
|--------|------------|-------|
| Distro PSIRTs | `StellaOps.Feedser.Source.Distro.*` | Debian, Red Hat, SUSE, Ubuntu connectors with NEVRA/EVR helpers. |
| Vendor PSIRTs | `StellaOps.Feedser.Source.Vndr.*` | Adobe, Apple, Cisco, Chromium, Microsoft, Oracle, VMware. |
| Regional CERTs | `StellaOps.Feedser.Source.Cert*`, `Source.Ru.*`, `Source.Ics.*`, `Source.Kisa` | Provide enrichment metadata while preserving vendor precedence. |
| OSS ecosystems | `StellaOps.Feedser.Source.Ghsa`, `Source.Osv`, `Source.Cve`, `Source.Kev`, `Source.Acsc`, `Source.Cccs`, `Source.Jvn` | Emit SemVer/alias-rich advisories. |
Each connector ships fixtures/tests under the matching `*.Tests` project.
---
##5·ModuleDetails
> _Focus on the Feedser-specific services that replace the legacy FeedMerge cron._
###5.1Feedser.Core
* Owns the fetch → parse → merge → export job pipeline and enforces deterministic
merge hashes (`merge_event`).
* Provides `JobSchedulerBuilder`, job coordinator, and telemetry scopes consumed
by the WebService and exporters.
###5.2Feedser.Storage.Mongo
* Bootstrapper creates collections/indexes (documents, dto, advisory, alias,
affected, merge_event, export_state, jobs, locks).
* Repository APIs surface full/delta advisory reads for exporters, plus
SourceState and job lease persistence.
###5.3Feedser.Exporter.Json / Feedser.Exporter.TrivyDb
* JSON exporter mirrors vuln-list layout with per-file digests and manifest.
* Trivy DB exporter shells or native-builds Bolt archives, optionally pushes OCI
layers, and records export cursors. Delta runs reuse unchanged blobs from the
previous full baseline, annotating `metadata.json` with `mode`, `baseExportId`,
`baseManifestDigest`, `resetBaseline`, and `delta.changedFiles[]`/`delta.removedPaths[]`.
ORAS pushes honour `publishFull` / `publishDelta`, and offline bundles respect
`includeFull` / `includeDelta` for air-gapped syncs.
###5.4Feedser.WebService
* Minimal API host exposing `/health`, `/ready`, `/jobs` and wiring telemetry.
* Loads restart-time plug-ins from `PluginBinaries/`, executes Mongo bootstrap,
and registers built-in connectors/exporters with the scheduler.
###5.5Plugin host & DI bridge
* `StellaOps.Plugin` + `StellaOps.DependencyInjection` provide the contracts and
helper routines for connectors/exporters to integrate with the WebService.
---
##6·Plug-ins & Agents
* **Plug-in discovery** restart-only; the WebService enumerates
`PluginBinaries/` (or configured directories) and executes the contained
`IDependencyInjectionRoutine` implementations.
* **Connector/exporter packages** each source/exporter can ship as a plug-in
assembly with its own options and HttpClient configuration, keeping the core
image minimal.
* **StellaOps CLI (agent)** new `StellaOps.Cli` module that exposes
`scanner`, `scan`, and `db` verbs (via System.CommandLine 2.0) to download
scanner container bundles, install them locally, execute scans against target
directories, automatically upload results, and trigger Feedser jobs (`db
fetch/merge/export`) aligned with the SBOM-first workflow described in
`AGENTS.md`.
* **Offline Kit** bundles Feedser plug-ins, JSON tree, Trivy DB, and export
manifests so air-gapped sites can load the latest vulnerability data without
outbound connectivity.
---
##7·Docker & Distribution Artefacts
| Artefact | Path / Identifier | Notes |
|----------|-------------------|-------|
| Feedser WebService image | `containers/feedser/Dockerfile` (built via CI) | Self-contained ASP.NET runtime hosting scheduler/endpoints. |
| Plugin bundle | `PluginBinaries/` | Mounted or baked-in assemblies for connectors/exporters. |
| Offline Kit tarball | Produced by CI release pipeline | Contains JSON tree, Trivy DB OCI layout, export manifest, and plug-ins. |
| Local dev compose | `scripts/` + future compose overlays | Developers can run MongoDB, Redis (optional), and WebService locally. |
---
##8·Performance Budget
| Scenario | Budget | Source |
|----------|--------|--------|
| Advisory upsert (large advisory) | ≤500ms/advisory | `AdvisoryStorePerformanceTests` (Mongo) |
| Advisory fetch (`GetRecent`) | ≤200ms/advisory | Same performance test harness |
| Advisory point lookup (`Find`) | ≤200ms/advisory | Same performance test harness |
| Bulk upsert/fetch cycle | ≤28s total for 30 large advisories | Same performance test harness |
| Feedser job scheduling | Deterministic cron execution via `JobSchedulerHostedService` | `StellaOps.Feedser.Core` tests |
| Trivy DB export | Deterministic digests across runs (ongoing TODO for end-to-end test) | `Exporter.TrivyDb` backlog |
Budgets are enforced in automated tests where available; outstanding TODO/DOING
items (see task boards) continue tracking gaps such as exporter determinism.
---
##9Testing
* Unit and integration tests live alongside each component (`*.Tests`).
* Shared fixtures come from `StellaOps.Feedser.Testing` and
`StellaOps.Feedser.Tests.Shared` (linked via `Directory.Build.props`).
* Integration suites use ephemeral MongoDB and Redis via Testcontainers to
validate end-to-end flow without external dependencies.
---

View File

@@ -200,7 +200,7 @@ Returns `202 Accepted` and `Location: /attest/{id}` for async verify.
## 3 StellaOps CLI (`stellaops-cli`)
The new CLI is built on **System.CommandLine2.0.0beta5** and mirrors the Feedser backend REST API.
The new CLI is built on **System.CommandLine2.0.0beta5** and mirrors the Concelier backend REST API.
Configuration follows the same precedence chain everywhere:
1. Environment variables (e.g. `API_KEY`, `STELLAOPS_BACKEND_URL`, `StellaOps:ApiKey`)
@@ -234,6 +234,11 @@ See `docs/dev/32_AUTH_CLIENT_GUIDE.md` for recommended profiles (online vs. air-
When running on an interactive terminal without explicit override flags, the CLI uses Spectre.Console prompts to let you choose per-run ORAS/offline bundle behaviour.
**Startup diagnostics**
- `stellaops-cli` now loads Authority plug-in manifests during startup (respecting `Authority:Plugins:*`) and surfaces analyzer warnings when a plug-in weakens the baseline password policy (minimum length **12** and all character classes required).
- Follow the log entrys config path and raise `passwordPolicy.minimumLength` to at least 12 while keeping `requireUppercase`, `requireLowercase`, `requireDigit`, and `requireSymbol` set to `true` to clear the warning; weakened overrides are treated as actionable security deviations.
**Logging & exit codes**
- Structured logging via `Microsoft.Extensions.Logging` with single-line console output (timestamps in UTC).
@@ -245,7 +250,7 @@ When running on an interactive terminal without explicit override flags, the CLI
- Downloads are verified against the `X-StellaOps-Digest` header (SHA-256). When `StellaOps:ScannerSignaturePublicKeyPath` points to a PEM-encoded RSA key, the optional `X-StellaOps-Signature` header is validated as well.
- Metadata for each bundle is written alongside the artefact (`*.metadata.json`) with digest, signature, source URL, and timestamps.
- Retry behaviour is controlled via `StellaOps:ScannerDownloadAttempts` (default **3** with exponential backoff).
- Successful `scan run` executions create timestamped JSON artefacts inside `ResultsDirectory` plus a `scan-run-*.json` metadata envelope documenting the runner, arguments, timing, and stdout/stderr. The artefact is posted back to Feedser automatically.
- Successful `scan run` executions create timestamped JSON artefacts inside `ResultsDirectory` plus a `scan-run-*.json` metadata envelope documenting the runner, arguments, timing, and stdout/stderr. The artefact is posted back to Concelier automatically.
#### Trivy DB export metadata (`metadata.json`)
@@ -260,18 +265,18 @@ When running on an interactive terminal without explicit override flags, the CLI
| `treeDigest` | string | Canonical SHA-256 digest of the JSON tree used to build the database. |
| `treeBytes` | number | Total bytes across exported JSON files. |
| `advisoryCount` | number | Count of advisories included in the export. |
| `exporterVersion` | string | Version stamp of `StellaOps.Feedser.Exporter.TrivyDb`. |
| `exporterVersion` | string | Version stamp of `StellaOps.Concelier.Exporter.TrivyDb`. |
| `builder` | object? | Raw metadata emitted by `trivy-db build` (version, update cadence, etc.). |
| `delta.changedFiles[]` | array | Present when `mode = delta`. Each entry lists `{ "path": "<relative json>", "length": <bytes>, "digest": "sha256:..." }`. |
| `delta.removedPaths[]` | array | Paths that existed in the previous manifest but were removed in the new run. |
When the planner opts for a delta run, the exporter copies unmodified blobs from the baseline layout identified by `baseManifestDigest`. Consumers that cache OCI blobs only need to fetch the `changedFiles` and the new manifest/metadata unless `resetBaseline` is true.
When pushing to ORAS, set `feedser:exporters:trivyDb:oras:publishFull` / `publishDelta` to control whether full or delta runs are copied to the registry. Offline bundles follow the analogous `includeFull` / `includeDelta` switches under `offlineBundle`.
When pushing to ORAS, set `concelier:exporters:trivyDb:oras:publishFull` / `publishDelta` to control whether full or delta runs are copied to the registry. Offline bundles follow the analogous `includeFull` / `includeDelta` switches under `offlineBundle`.
Example configuration (`appsettings.yaml`):
```yaml
feedser:
concelier:
exporters:
trivyDb:
oras:
@@ -288,7 +293,7 @@ feedser:
**Authentication**
- API key is sent as `Authorization: Bearer <token>` automatically when configured.
- Anonymous operation is permitted only when Feedser runs with
- Anonymous operation is permitted only when Concelier runs with
`authority.allowAnonymousFallback: true`. This flag is temporary—plan to disable
it before **2025-12-31 UTC** so bearer tokens become mandatory.
@@ -298,7 +303,7 @@ Authority-backed auth workflow:
3. Execute CLI commands as normal—the backend client injects the cached bearer token automatically and retries on transient 401/403 responses with operator guidance.
4. Inspect the cache with `stellaops-cli auth status` (shows expiry, scope, mode) or clear it via `stellaops-cli auth logout`.
5. Run `stellaops-cli auth whoami` to dump token subject, audience, issuer, scopes, and remaining lifetime (verbose mode prints additional claims).
6. Expect Feedser to emit audit logs for each `/jobs*` request showing `subject`,
6. Expect Concelier to emit audit logs for each `/jobs*` request showing `subject`,
`clientId`, `scopes`, `status`, and whether network bypass rules were applied.
Tokens live in `~/.stellaops/tokens` unless `StellaOps:Authority:TokenCacheDirectory` overrides it. Cached tokens are reused offline until they expire; the CLI surfaces clear errors if refresh fails.
@@ -309,7 +314,7 @@ Tokens live in `~/.stellaops/tokens` unless `StellaOps:Authority:TokenCacheDirec
{
"StellaOps": {
"ApiKey": "your-api-token",
"BackendUrl": "https://feedser.example.org",
"BackendUrl": "https://concelier.example.org",
"ScannerCacheDirectory": "scanners",
"ResultsDirectory": "results",
"DefaultRunner": "docker",
@@ -317,11 +322,11 @@ Tokens live in `~/.stellaops/tokens` unless `StellaOps:Authority:TokenCacheDirec
"ScannerDownloadAttempts": 3,
"Authority": {
"Url": "https://authority.example.org",
"ClientId": "feedser-cli",
"ClientId": "concelier-cli",
"ClientSecret": "REDACTED",
"Username": "",
"Password": "",
"Scope": "feedser.jobs.trigger",
"Scope": "concelier.jobs.trigger",
"TokenCacheDirectory": ""
}
}

View File

@@ -1,6 +1,6 @@
# 10 · Feedser + CLI Quickstart
# 10 · Concelier + CLI Quickstart
This guide walks through configuring the Feedser web service and the `stellaops-cli`
This guide walks through configuring the Concelier web service and the `stellaops-cli`
tool so an operator can ingest advisories, merge them, and publish exports from a
single workstation. It focuses on deployment-facing surfaces only (configuration,
runtime wiring, CLI usage) and leaves connector/internal customization for later.
@@ -16,39 +16,39 @@ runtime wiring, CLI usage) and leaves connector/internal customization for later
- Optional: Docker/Podman runtime if you plan to run scanners locally
> **Tip** air-gapped installs should preload `trivy-db` and `oras` binaries into the
> runner image since Feedser never fetches them dynamically.
> runner image since Concelier never fetches them dynamically.
---
## 1 · Configure Feedser
## 1 · Configure Concelier
1. Copy the sample config to the expected location (CI/CD pipelines can stamp values
into this file during deployment—see the “Deployment automation” note below):
```bash
mkdir -p etc
cp etc/feedser.yaml.sample etc/feedser.yaml
cp etc/concelier.yaml.sample etc/concelier.yaml
```
2. Edit `etc/feedser.yaml` and update the MongoDB DSN (and optional database name).
2. Edit `etc/concelier.yaml` and update the MongoDB DSN (and optional database name).
The default template configures plug-in discovery to look in `PluginBinaries/`
and disables remote telemetry exporters by default.
3. (Optional) Override settings via environment variables. All keys are prefixed with
`FEEDSER_`. Example:
`CONCELIER_`. Example:
```bash
export FEEDSER_STORAGE__DSN="mongodb://user:pass@mongo:27017/feedser"
export FEEDSER_TELEMETRY__ENABLETRACING=false
export CONCELIER_STORAGE__DSN="mongodb://user:pass@mongo:27017/concelier"
export CONCELIER_TELEMETRY__ENABLETRACING=false
```
4. Start the web service from the repository root:
```bash
dotnet run --project src/StellaOps.Feedser.WebService
dotnet run --project src/StellaOps.Concelier.WebService
```
On startup Feedser validates the options, boots MongoDB indexes, loads plug-ins,
On startup Concelier validates the options, boots MongoDB indexes, loads plug-ins,
and exposes:
- `GET /health` returns service status and telemetry settings
@@ -90,7 +90,7 @@ defaults live in `src/StellaOps.Cli/appsettings.json` and expect overrides at ru
| Setting | Environment variable | Default | Purpose |
| ------- | -------------------- | ------- | ------- |
| `BackendUrl` | `STELLAOPS_BACKEND_URL` | _empty_ | Base URL of the Feedser web service |
| `BackendUrl` | `STELLAOPS_BACKEND_URL` | _empty_ | Base URL of the Concelier web service |
| `ApiKey` | `API_KEY` | _empty_ | Reserved for legacy key auth; leave empty when using Authority |
| `ScannerCacheDirectory` | `STELLAOPS_SCANNER_CACHE_DIRECTORY` | `scanners` | Local cache folder |
| `ResultsDirectory` | `STELLAOPS_RESULTS_DIRECTORY` | `results` | Where scan outputs are written |
@@ -99,7 +99,7 @@ defaults live in `src/StellaOps.Cli/appsettings.json` and expect overrides at ru
| `Authority.ClientSecret` | `STELLAOPS_AUTHORITY_CLIENT_SECRET` | _empty_ | Client secret (omit when using username/password grant) |
| `Authority.Username` | `STELLAOPS_AUTHORITY_USERNAME` | _empty_ | Username for password grant flows |
| `Authority.Password` | `STELLAOPS_AUTHORITY_PASSWORD` | _empty_ | Password for password grant flows |
| `Authority.Scope` | `STELLAOPS_AUTHORITY_SCOPE` | `feedser.jobs.trigger` | OAuth scope requested for backend operations |
| `Authority.Scope` | `STELLAOPS_AUTHORITY_SCOPE` | `concelier.jobs.trigger` | OAuth scope requested for backend operations |
| `Authority.TokenCacheDirectory` | `STELLAOPS_AUTHORITY_TOKEN_CACHE_DIR` | `~/.stellaops/tokens` | Directory that persists cached tokens |
| `Authority.Resilience.EnableRetries` | `STELLAOPS_AUTHORITY_ENABLE_RETRIES` | `true` | Toggle Polly retry handler for Authority HTTP calls |
| `Authority.Resilience.RetryDelays` | `STELLAOPS_AUTHORITY_RETRY_DELAYS` | `1s,2s,5s` | Comma- or space-separated backoff delays (hh:mm:ss) |
@@ -112,7 +112,7 @@ Example bootstrap:
export STELLAOPS_BACKEND_URL="http://localhost:5000"
export STELLAOPS_RESULTS_DIRECTORY="$HOME/.stellaops/results"
export STELLAOPS_AUTHORITY_URL="https://authority.local"
export STELLAOPS_AUTHORITY_CLIENT_ID="feedser-cli"
export STELLAOPS_AUTHORITY_CLIENT_ID="concelier-cli"
export STELLAOPS_AUTHORITY_CLIENT_SECRET="s3cr3t"
dotnet run --project src/StellaOps.Cli -- db merge
@@ -161,7 +161,7 @@ rely on environment variables for ephemeral runners.
dotnet run --project src/StellaOps.Cli -- db export --format trivy-db --delta
```
Feedser always produces a deterministic OCI layout. The first run after a clean
Concelier always produces a deterministic OCI layout. The first run after a clean
bootstrap emits a **full** baseline; subsequent `--delta` runs reuse the previous
baselines blobs when only JSON manifests change. If the exporter detects that a
prior delta is still active (i.e., `LastDeltaDigest` is recorded) it automatically
@@ -175,7 +175,7 @@ rely on environment variables for ephemeral runners.
while reusing the previous layer blob.
```bash
export_root=${FEEDSER_EXPORT_ROOT:-exports/trivy}
export_root=${CONCELIER_EXPORT_ROOT:-exports/trivy}
base=$(ls -1d "$export_root"/* | sort | tail -n2 | head -n1)
delta=$(ls -1d "$export_root"/* | sort | tail -n1)
@@ -209,10 +209,10 @@ a problem document.
## 4 · Verification Checklist
- Feedser `/health` returns `"status":"healthy"` and Storage bootstrap is marked
- Concelier `/health` returns `"status":"healthy"` and Storage bootstrap is marked
complete after startup.
- CLI commands return HTTP 202 with a `Location` header (job tracking URL) when
triggering Feedser jobs.
triggering Concelier jobs.
- Export artefacts are materialised under the configured output directories and
their manifests record digests.
- MongoDB contains the expected `document`, `dto`, `advisory`, and `export_state`
@@ -222,7 +222,7 @@ a problem document.
## 5 · Deployment Automation
- Treat `etc/feedser.yaml.sample` as the canonical template. CI/CD should copy it to
- Treat `etc/concelier.yaml.sample` as the canonical template. CI/CD should copy it to
the deployment artifact and replace placeholders (DSN, telemetry endpoints, cron
overrides) with environment-specific secrets.
- Keep secret material (Mongo credentials, OTLP tokens) outside of the repository;
@@ -238,15 +238,15 @@ a problem document.
`authority.enabled: true` while keeping `authority.allowAnonymousFallback: true`
to observe logs, then flip it to `false` before 2025-12-31 UTC to enforce tokens.
- Automate the workflow above via CI/CD (compose stack or Kubernetes CronJobs).
- Pair with the Feedser connector teams when enabling additional sources so their
- Pair with the Concelier connector teams when enabling additional sources so their
module-specific requirements are pulled in safely.
---
## 6 · Authority Integration
- Feedser now authenticates callers through StellaOps Authority using OAuth 2.0
resource server flows. Populate the `authority` block in `feedser.yaml`:
- Concelier now authenticates callers through StellaOps Authority using OAuth 2.0
resource server flows. Populate the `authority` block in `concelier.yaml`:
```yaml
authority:
@@ -254,20 +254,20 @@ a problem document.
allowAnonymousFallback: false # keep true only during the staged rollout window
issuer: "https://authority.example.org"
audiences:
- "api://feedser"
- "api://concelier"
requiredScopes:
- "feedser.jobs.trigger"
clientId: "feedser-jobs"
clientSecretFile: "../secrets/feedser-jobs.secret"
- "concelier.jobs.trigger"
clientId: "concelier-jobs"
clientSecretFile: "../secrets/concelier-jobs.secret"
clientScopes:
- "feedser.jobs.trigger"
- "concelier.jobs.trigger"
bypassNetworks:
- "127.0.0.1/32"
- "::1/128"
```
- Store the client secret outside of source control. Either provide it via
`authority.clientSecret` (environment variable `FEEDSER_AUTHORITY__CLIENTSECRET`)
`authority.clientSecret` (environment variable `CONCELIER_AUTHORITY__CLIENTSECRET`)
or point `authority.clientSecretFile` to a file mounted at runtime.
- Cron jobs running on the same host can keep using the API thanks to the loopback
bypass mask. Add additional CIDR ranges as needed; every bypass is logged.
@@ -275,15 +275,15 @@ a problem document.
variables such as:
```bash
export FEEDSER_AUTHORITY__ENABLED=true
export FEEDSER_AUTHORITY__ALLOWANONYMOUSFALLBACK=false
export FEEDSER_AUTHORITY__ISSUER="https://authority.example.org"
export FEEDSER_AUTHORITY__CLIENTID="feedser-jobs"
export FEEDSER_AUTHORITY__CLIENTSECRETFILE="/var/run/secrets/feedser/authority-client"
export CONCELIER_AUTHORITY__ENABLED=true
export CONCELIER_AUTHORITY__ALLOWANONYMOUSFALLBACK=false
export CONCELIER_AUTHORITY__ISSUER="https://authority.example.org"
export CONCELIER_AUTHORITY__CLIENTID="concelier-jobs"
export CONCELIER_AUTHORITY__CLIENTSECRETFILE="/var/run/secrets/concelier/authority-client"
```
- CLI commands already pass `Authorization` headers when credentials are supplied.
Configure the CLI with matching Authority settings (`docs/09_API_CLI_REFERENCE.md`)
so that automation can obtain tokens with the same client credentials. Feedser
so that automation can obtain tokens with the same client credentials. Concelier
logs every job request with the client ID, subject (if present), scopes, and
a `bypass` flag so operators can audit cron traffic.

View File

@@ -3,14 +3,14 @@
> **Status:** Drafted 2025-10-12 (CORE5B.DOC / DOC1.AUTH) aligns with Authority revocation store, JWKS rotation, and bootstrap endpoints delivered in Sprint1.
## 1. Purpose
The **StellaOps Authority** service issues OAuth2/OIDC tokens for every StellaOps module (Feedser, Backend, Agent, Zastava) and exposes the policy controls required in sovereign/offline environments. Authority is built as a minimal ASP.NET host that:
The **StellaOps Authority** service issues OAuth2/OIDC tokens for every StellaOps module (Concelier, Backend, Agent, Zastava) and exposes the policy controls required in sovereign/offline environments. Authority is built as a minimal ASP.NET host that:
- brokers password, client-credentials, and device-code flows through pluggable identity providers;
- persists access/refresh/device tokens in MongoDB with deterministic schemas for replay analysis and air-gapped audit copies;
- distributes revocation bundles and JWKS material so downstream services can enforce lockouts without direct database access;
- offers bootstrap APIs for first-run provisioning and key rotation without redeploying binaries.
Authority is deployed alongside Feedser in air-gapped environments and never requires outbound internet access. All trusted metadata (OpenIddict discovery, JWKS, revocation bundles) is cacheable, signed, and reproducible.
Authority is deployed alongside Concelier in air-gapped environments and never requires outbound internet access. All trusted metadata (OpenIddict discovery, JWKS, revocation bundles) is cacheable, signed, and reproducible.
## 2. Component Architecture
Authority is composed of five cooperating subsystems:
@@ -46,7 +46,7 @@ Authority persists every issued token in MongoDB so operators can audit or revok
- **Expiry maintenance:** `AuthorityTokenStore.DeleteExpiredAsync` prunes non-revoked tokens past their `expiresAt` timestamp. Operators should schedule this in maintenance windows if large volumes of tokens are issued.
### Expectations for resource servers
Resource servers (Feedser WebService, Backend, Agent) **must not** assume in-memory caches are authoritative. They should:
Resource servers (Concelier WebService, Backend, Agent) **must not** assume in-memory caches are authoritative. They should:
- cache `/jwks` and `/revocations/export` responses within configured lifetimes;
- honour `revokedReason` metadata when shaping audit trails;
@@ -67,12 +67,13 @@ Authority centralises revocation in `authority_revocations` with deterministic c
**Export surfaces** (deterministic output, suitable for Offline Kit):
- CLI: `stella auth revoke export --output ./out` writes `revocation-bundle.json`, `.jws`, `.sha256`.
- Verification: `stella auth revoke verify --bundle <path> --signature <path> --key <path>` validates detached JWS signatures before distribution, selecting the crypto provider advertised in the detached header (see `docs/security/revocation-bundle.md`).
- API: `GET /internal/revocations/export` (requires bootstrap API key) returns the same payload.
- Verification: `stella auth revoke verify` validates schema, digest, and detached JWS using cached JWKS or offline keys.
- Verification: `stella auth revoke verify` validates schema, digest, and detached JWS using cached JWKS or offline keys, automatically preferring the hinted provider (libsodium builds honour `provider=libsodium`; other builds fall back to the managed provider).
**Consumer guidance:**
1. Mirror `revocation-bundle.json*` alongside Feedser exports. Offline agents fetch both over the existing update channel.
1. Mirror `revocation-bundle.json*` alongside Concelier exports. Offline agents fetch both over the existing update channel.
2. Use bundle `sequence` and `bundleId` to detect replay or monotonicity regressions. Ignore bundles with older sequence numbers unless `bundleId` changes and `issuedAt` advances.
3. Treat `revokedReason` taxonomy as machine-friendly codes (`compromised`, `rotation`, `policy`, `lifecycle`). Translating to human-readable logs is the consumers responsibility.
@@ -153,7 +154,7 @@ All administrative calls emit `AuthEventRecord` entries enriched with correlatio
## 9. Operational Checklist
- [ ] Protect the bootstrap API key and disable bootstrap endpoints (`bootstrap.enabled: false`) once initial setup is complete.
- [ ] Schedule `stella auth revoke export` (or `/internal/revocations/export`) at the same cadence as Feedser exports so bundles remain in lockstep.
- [ ] Schedule `stella auth revoke export` (or `/internal/revocations/export`) at the same cadence as Concelier exports so bundles remain in lockstep.
- [ ] Rotate signing keys before expiration; keep at least one retired key until all cached bundles/tokens signed with it have expired.
- [ ] Monitor `/health` and `/ready` plus rate-limiter metrics to detect plugin outages early.
- [ ] Ensure downstream services cache JWKS and revocation bundles within tolerances; stale caches risk accepting revoked tokens.

View File

@@ -81,7 +81,7 @@ cosign verify \
## 5·Privatefeed mirrors 🌐
The **Feedser (vulnerability ingest/merge/export service)** provides signed JSON and Trivy DB snapshots that merge:
The **Concelier (vulnerability ingest/merge/export service)** provides signed JSON and Trivy DB snapshots that merge:
* OSV + GHSA
* (optional) NVD 2.0, CNNVD, CNVD, ENISA, JVN and BDU regionals

View File

@@ -20,7 +20,7 @@ open a PR and append it alphabetically.*
| **ADR** | *Architecture Decision Record* lightweight Markdown file that captures one irreversible design decision. | ADR template lives at `/docs/adr/` |
| **AIRE** | *AI Risk Evaluator* optional Plus/Pro plugin that suggests mute rules using an ONNX model. | Commercial feature |
| **AzurePipelines** | CI/CD service in Microsoft Azure DevOps. | Recipe in Pipeline Library |
| **BDU** | Russian (FSTEC) national vulnerability database: *База данных уязвимостей*. | Merged with NVD by Feedser (vulnerability ingest/merge/export service) |
| **BDU** | Russian (FSTEC) national vulnerability database: *База данных уязвимостей*. | Merged with NVD by Concelier (vulnerability ingest/merge/export service) |
| **BuildKit** | Modern Docker build engine with caching and concurrency. | Needed for layer cache patterns |
| **CI** | *Continuous Integration* automated build/test pipeline. | Stella integrates via CLI |
| **Cosign** | Opensource Sigstore tool that signs & verifies container images **and files**. | Images & OUK tarballs |
@@ -36,7 +36,7 @@ open a PR and append it alphabetically.*
| **Digest (image)** | SHA256 hash uniquely identifying a container image or layer. | Pin digests for reproducible builds |
| **DockerinDocker (DinD)** | Running Docker daemon inside a CI container. | Used in GitHub / GitLab recipes |
| **DTO** | *Data Transfer Object* C# record serialised to JSON. | Schemas in doc 11 |
| **Feedser** | Vulnerability ingest/merge/export service consolidating OVN, GHSA, NVD 2.0, CNNVD, CNVD, ENISA, JVN and BDU feeds into the canonical MongoDB store and export artifacts. | Cron default `01* * *` |
| **Concelier** | Vulnerability ingest/merge/export service consolidating OVN, GHSA, NVD 2.0, CNNVD, CNVD, ENISA, JVN and BDU feeds into the canonical MongoDB store and export artifacts. | Cron default `01* * *` |
| **FSTEC** | Russian regulator issuing SOBIT certificates. | Pro GA target |
| **Gitea** | Selfhosted Git service mirrors GitHub repo. | OSS hosting |
| **GOST TLS** | TLS ciphersuites defined by Russian GOST R 34.102012 / 34.112012. | Provided by `OpenSslGost` or CryptoPro |

View File

@@ -145,28 +145,28 @@ cosign verify ghcr.io/stellaops/backend@sha256:<DIGEST> \
| Audit events | Redis stream audit; export daily to SIEM |
| Alert rules | Feed age 48h, P95 walltime>5s, Redis used memory>75% |
### 7.1Feedser authorization audits
### 7.1Concelier authorization audits
- Enable the Authority integration for Feedser (`authority.enabled=true`). Keep
- Enable the Authority integration for Concelier (`authority.enabled=true`). Keep
`authority.allowAnonymousFallback` set to `true` only during migration and plan
to disable it before **2025-12-31 UTC** so the `/jobs*` surface always demands
a bearer token.
- Store the Authority client secret using Docker/Kubernetes secrets and point
`authority.clientSecretFile` at the mounted path; the value is read at startup
and never logged.
- Watch the `Feedser.Authorization.Audit` logger. Each entry contains the HTTP
- Watch the `Concelier.Authorization.Audit` logger. Each entry contains the HTTP
status, subject, client ID, scopes, remote IP, and a boolean `bypass` flag
showing whether a network bypass CIDR allowed the request. Configure your SIEM
to alert when unauthenticated requests (`status=401`) appear with
`bypass=true`, or when unexpected scopes invoke job triggers.
Detailed monitoring and response guidance lives in `docs/ops/feedser-authority-audit-runbook.md`.
Detailed monitoring and response guidance lives in `docs/ops/concelier-authority-audit-runbook.md`.
## 8Update & patch strategy
| Layer | Cadence | Method |
| -------------------- | -------------------------------------------------------- | ------------------------------ |
| Backend & CLI images | Monthly or CVEdriven docker pull + docker compose up -d |
| Trivy DB | 24h scheduler via Feedser (vulnerability ingest/merge/export service) | configurable via Feedser scheduler options |
| Trivy DB | 24h scheduler via Concelier (vulnerability ingest/merge/export service) | configurable via Concelier scheduler options |
| Docker Engine | vendor LTS | distro package manager |
| Host OS | security repos enabled | unattendedupgrades |

View File

@@ -16,7 +16,7 @@ contributors who need to extend coverage or diagnose failures.
| **1. Unit** | `xUnit` (<code>dotnet test</code>) | `*.Tests.csproj` | per PR / push |
| **2. Propertybased** | `FsCheck` | `SbomPropertyTests` | per PR |
| **3. Integration (API)** | `Testcontainers` suite | `test/Api.Integration` | per PR + nightly |
| **4. Integration (DB-merge)** | in-memory Mongo + Redis | `Feedser.Integration` (vulnerability ingest/merge/export service) | per PR |
| **4. Integration (DB-merge)** | in-memory Mongo + Redis | `Concelier.Integration` (vulnerability ingest/merge/export service) | per PR |
| **5. Contract (gRPC)** | `Buf breaking` | `buf.yaml` files | per PR |
| **6. Frontend unit** | `Jest` | `ui/src/**/*.spec.ts` | per PR |
| **7. Frontend E2E** | `Playwright` | `ui/e2e/**` | nightly |
@@ -59,17 +59,17 @@ The script spins up MongoDB/Redis via Testcontainers and requires:
---
### Feedser OSV↔GHSA parity fixtures
### Concelier OSV↔GHSA parity fixtures
The Feedser connector suite includes a regression test (`OsvGhsaParityRegressionTests`)
The Concelier connector suite includes a regression test (`OsvGhsaParityRegressionTests`)
that checks a curated set of GHSA identifiers against OSV responses. The fixture
snapshots live in `src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/` and are kept
snapshots live in `src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/` and are kept
deterministic so the parity report remains reproducible.
To refresh the fixtures when GHSA/OSV payloads change:
1. Ensure outbound HTTPS access to `https://api.osv.dev` and `https://api.github.com`.
2. Run `UPDATE_PARITY_FIXTURES=1 dotnet test src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj`.
2. Run `UPDATE_PARITY_FIXTURES=1 dotnet test src/StellaOps.Concelier.Connector.Osv.Tests/StellaOps.Concelier.Connector.Osv.Tests.csproj`.
3. Commit the regenerated `osv-ghsa.*.json` files that the test emits (raw snapshots and canonical advisories).
The regen flow logs `[Parity]` messages and normalises `recordedAt` timestamps so the
@@ -88,7 +88,7 @@ flowchart LR
I1 --> FE[Jest]
FE --> E2E[Playwright]
E2E --> Lighthouse
Lighthouse --> INTEG2[Feedser]
Lighthouse --> INTEG2[Concelier]
INTEG2 --> LOAD[k6]
LOAD --> CHAOS[pumba]
CHAOS --> RELEASE[Attestation diff]

View File

@@ -76,51 +76,51 @@ UI: [https://\&lt;host\&gt;:8443](https://&lt;host&gt;:8443) (selfsigned cert
> `stella-ops:latest` with the immutable digest printed by
> `docker images --digests`.
### 1.1 · Feedser authority configuration
### 1.1 · Concelier authority configuration
The Feedser container reads configuration from `etc/feedser.yaml` plus
`FEEDSER_` environment variables. To enable the new Authority integration:
The Concelier container reads configuration from `etc/concelier.yaml` plus
`CONCELIER_` environment variables. To enable the new Authority integration:
1. Add the following keys to `.env` (replace values for your environment):
```bash
FEEDSER_AUTHORITY__ENABLED=true
FEEDSER_AUTHORITY__ALLOWANONYMOUSFALLBACK=true # temporary rollout only
FEEDSER_AUTHORITY__ISSUER="https://authority.internal"
FEEDSER_AUTHORITY__AUDIENCES__0="api://feedser"
FEEDSER_AUTHORITY__REQUIREDSCOPES__0="feedser.jobs.trigger"
FEEDSER_AUTHORITY__CLIENTID="feedser-jobs"
FEEDSER_AUTHORITY__CLIENTSECRETFILE="/run/secrets/feedser_authority_client"
FEEDSER_AUTHORITY__BYPASSNETWORKS__0="127.0.0.1/32"
FEEDSER_AUTHORITY__BYPASSNETWORKS__1="::1/128"
FEEDSER_AUTHORITY__RESILIENCE__ENABLERETRIES=true
FEEDSER_AUTHORITY__RESILIENCE__RETRYDELAYS__0="00:00:01"
FEEDSER_AUTHORITY__RESILIENCE__RETRYDELAYS__1="00:00:02"
FEEDSER_AUTHORITY__RESILIENCE__RETRYDELAYS__2="00:00:05"
FEEDSER_AUTHORITY__RESILIENCE__ALLOWOFFLINECACHEFALLBACK=true
FEEDSER_AUTHORITY__RESILIENCE__OFFLINECACHETOLERANCE="00:10:00"
CONCELIER_AUTHORITY__ENABLED=true
CONCELIER_AUTHORITY__ALLOWANONYMOUSFALLBACK=true # temporary rollout only
CONCELIER_AUTHORITY__ISSUER="https://authority.internal"
CONCELIER_AUTHORITY__AUDIENCES__0="api://concelier"
CONCELIER_AUTHORITY__REQUIREDSCOPES__0="concelier.jobs.trigger"
CONCELIER_AUTHORITY__CLIENTID="concelier-jobs"
CONCELIER_AUTHORITY__CLIENTSECRETFILE="/run/secrets/concelier_authority_client"
CONCELIER_AUTHORITY__BYPASSNETWORKS__0="127.0.0.1/32"
CONCELIER_AUTHORITY__BYPASSNETWORKS__1="::1/128"
CONCELIER_AUTHORITY__RESILIENCE__ENABLERETRIES=true
CONCELIER_AUTHORITY__RESILIENCE__RETRYDELAYS__0="00:00:01"
CONCELIER_AUTHORITY__RESILIENCE__RETRYDELAYS__1="00:00:02"
CONCELIER_AUTHORITY__RESILIENCE__RETRYDELAYS__2="00:00:05"
CONCELIER_AUTHORITY__RESILIENCE__ALLOWOFFLINECACHEFALLBACK=true
CONCELIER_AUTHORITY__RESILIENCE__OFFLINECACHETOLERANCE="00:10:00"
```
Store the client secret outside source control (Docker secrets, mounted file,
or Kubernetes Secret). Feedser loads the secret during post-configuration, so
or Kubernetes Secret). Concelier loads the secret during post-configuration, so
the value never needs to appear in the YAML template.
Connected sites can keep the retry ladder short (1s,2s,5s) so job triggers fail fast when Authority is down. For airgapped or intermittently connected deployments, extend `RESILIENCE__OFFLINECACHETOLERANCE` (e.g. `00:30:00`) so cached discovery/JWKS data remains valid while the Offline Kit synchronises upstream changes.
2. Redeploy Feedser:
2. Redeploy Concelier:
```bash
docker compose --env-file .env -f docker-compose.stella-ops.yml up -d feedser
docker compose --env-file .env -f docker-compose.stella-ops.yml up -d concelier
```
3. Tail the logs: `docker compose logs -f feedser`. Successful `/jobs*` calls now
emit `Feedser.Authorization.Audit` entries with `route`, `status`, `subject`,
3. Tail the logs: `docker compose logs -f concelier`. Successful `/jobs*` calls now
emit `Concelier.Authorization.Audit` entries with `route`, `status`, `subject`,
`clientId`, `scopes`, `bypass`, and `remote` fields. 401 denials keep the same
shape—watch for `bypass=True`, which indicates a bypass CIDR accepted an anonymous
call. See `docs/ops/feedser-authority-audit-runbook.md` for a full audit/alerting checklist.
call. See `docs/ops/concelier-authority-audit-runbook.md` for a full audit/alerting checklist.
> **Enforcement deadline** keep `FEEDSER_AUTHORITY__ALLOWANONYMOUSFALLBACK=true`
> only while validating the rollout. Set it to `false` (and restart Feedser)
> **Enforcement deadline** keep `CONCELIER_AUTHORITY__ALLOWANONYMOUSFALLBACK=true`
> only while validating the rollout. Set it to `false` (and restart Concelier)
> before **2025-12-31 UTC** to require tokens in production.
---

View File

@@ -10,32 +10,50 @@
The **Offline Update Kit** packages everything StellaOps needs to run on a
completely isolated network:
| Component | Contents |
|-----------|----------|
| **Merged vulnerability feeds** | OSV, GHSA plus optional NVD 2.0, CNNVD, CNVD, ENISA, JVN and BDU |
| **Container images** | `stella-ops`, *Zastava* sidecar (x8664 &arm64) |
| **Provenance** | Cosign signature, SPDX 2.3 SBOM, intoto SLSA attestation |
| **Delta patches** | Daily diff bundles keep size \<350MB |
*Scanner core:* C# 12 on **.NET{{ dotnet }}**.
*Imports are idempotent and atomic — no service downtime.*
| Component | Contents |
|-----------|----------|
| **Merged vulnerability feeds** | OSV, GHSA plus optional NVD 2.0, CNNVD, CNVD, ENISA, JVN and BDU |
| **Container images** | `stella-ops`, *Zastava* sidecar (x8664 &arm64) |
| **Provenance** | Cosign signature, SPDX 2.3 SBOM, intoto SLSA attestation |
| **Attested manifest** | `offline-manifest.json` + detached JWS covering bundle metadata, signed during export. |
| **Delta patches** | Daily diff bundles keep size \<350MB |
**RU BDU note:** ship the official Russian Trusted Root/Sub CA bundle (`certificates/russian_trusted_bundle.pem`) inside the kit so `concelier:httpClients:source.bdu:trustedRootPaths` can resolve it when the service runs in an airgapped network. Drop the most recent `vulxml.zip` alongside the kit if operators need a cold-start cache.
*Scanner core:* C# 12 on **.NET{{ dotnet }}**.
*Imports are idempotent and atomic — no service downtime.*
---
## 1·Download & verify
```bash
curl -LO https://get.stella-ops.org/ouk/stella-ops-offline-kit-<DATE>.tgz
curl -LO https://get.stella-ops.org/ouk/stella-ops-offline-kit-<DATE>.tgz.sig
cosign verify-blob \
--key https://stella-ops.org/keys/cosign.pub \
--signature stella-ops-offline-kit-<DATE>.tgz.sig \
stella-ops-offline-kit-<DATE>.tgz
```bash
curl -LO https://get.stella-ops.org/ouk/stella-ops-offline-kit-<DATE>.tgz
curl -LO https://get.stella-ops.org/ouk/stella-ops-offline-kit-<DATE>.tgz.sig
curl -LO https://get.stella-ops.org/ouk/offline-manifest-<DATE>.json
curl -LO https://get.stella-ops.org/ouk/offline-manifest-<DATE>.json.jws
cosign verify-blob \
--key https://stella-ops.org/keys/cosign.pub \
--signature stella-ops-offline-kit-<DATE>.tgz.sig \
stella-ops-offline-kit-<DATE>.tgz
````
Verification prints **OK** and the SHA256 digest; crosscheck against the
[changelog](https://git.stella-ops.org/stella-ops/offline-kit/-/releases).
Verification prints **OK** and the SHA256 digest; crosscheck against the
[changelog](https://git.stella-ops.org/stella-ops/offline-kit/-/releases).
Validate the attested manifest before distribution:
```bash
cosign verify-blob \
--key https://stella-ops.org/keys/cosign.pub \
--signature offline-manifest-<DATE>.json.jws \
offline-manifest-<DATE>.json
jq '.artifacts[] | {name, sha256, size, capturedAt}' offline-manifest-<DATE>.json
```
The manifest enumerates every artefact (`name`, `sha256`, `size`, `capturedAt`) and is signed with the same key registry as Authority revocation bundles. Operators can ship the manifest alongside the tarball so downstream mirrors can re-verify without unpacking the kit.
---
@@ -89,6 +107,7 @@ See the detailed rules in
## 6·Related documentation
* **Install guide:** `/install/#air-gapped`
* **Sovereign mode rationale:** `/sovereign/`
* **Security policy:** `/security/#reporting-a-vulnerability`
* **Install guide:** `/install/#air-gapped`
* **Sovereign mode rationale:** `/sovereign/`
* **Security policy:** `/security/#reporting-a-vulnerability`
* **CERT-Bund snapshots:** `python tools/certbund_offline_snapshot.py --help` (see `docs/ops/concelier-certbund-operations.md`)

View File

@@ -32,7 +32,7 @@ why the system leans *monolithplusplugins*, and where extension points
graph TD
A(API Gateway)
B1(Scanner Core<br/>.NET latest LTS)
B2(Feedser service\n(vuln ingest/merge/export))
B2(Concelier service\n(vuln ingest/merge/export))
B3(Policy Engine OPA)
C1(Redis 7)
C2(MongoDB 7)
@@ -53,7 +53,7 @@ graph TD
| ---------------------------- | --------------------- | ---------------------------------------------------- |
| **API Gateway** | ASP.NET Minimal API | Auth (JWT), quotas, request routing |
| **Scanner Core** | C# 12, Polly | Layer diffing, SBOM generation, vuln correlation |
| **Feedser (vulnerability ingest/merge/export service)** | C# source-gen workers | Consolidate NVD + regional CVE feeds into the canonical MongoDB store and drive JSON / Trivy DB exports |
| **Concelier (vulnerability ingest/merge/export service)** | C# source-gen workers | Consolidate NVD + regional CVE feeds into the canonical MongoDB store and drive JSON / Trivy DB exports |
| **Policy Engine** | OPA (Rego) | admission decisions, custom org rules |
| **Redis 7** | KeyDB compatible | LRU cache, quota counters |
| **MongoDB 7** | WiredTiger | SBOM & findings storage |
@@ -121,7 +121,7 @@ Hotplugging is deferred until after v1.0 for security review.
Although the default deployment is a single container, each subservice can be
extracted:
* Feedser → standalone cron pod.
* Concelier → standalone cron pod.
* Policy Engine → sidecar (OPA) with gRPC contract.
* ResultSink → queue worker (RabbitMQ or Azure Service Bus).

View File

@@ -5,12 +5,13 @@ Produce and maintain offline-friendly documentation for StellaOps modules, cover
## Scope Highlights
- Authority docs (`docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md`, upcoming `docs/11_AUTHORITY.md`).
- Feedser quickstarts, CLI guides, Offline Kit manuals.
- Concelier quickstarts, CLI guides, Offline Kit manuals.
- Release notes and migration playbooks.
## Operating Principles
- Keep guides deterministic and in sync with shipped configuration samples.
- Prefer tables/checklists for operator steps; flag security-sensitive actions.
- When work involves a specific `StellaOps.<Component>` project, consult both `docs/07_HIGH_LEVEL_ARCHITECTURE.md` and the matching dossier `docs/ARCHITECTURE_<COMPONENT>.md` before drafting or editing content.
- Update `docs/TASKS.md` whenever work items change status (TODO/DOING/REVIEW/DONE/BLOCKED).
## Coordination

View File

@@ -0,0 +1,384 @@
# component_architecture_attestor.md — **StellaOps Attestor** (2025Q4)
> **Scope.** Implementationready architecture for the **Attestor**: the service that **submits** DSSE envelopes to **Rekor v2**, retrieves/validates inclusion proofs, caches results, and exposes verification APIs. It accepts DSSE **only** from the **Signer** over mTLS, enforces chainoftrust to StellaOps roots, and returns `{uuid, index, proof, logURL}` to calling services (Scanner.WebService for SBOMs; backend for final reports; Excititor exports when configured).
---
## 0) Mission & boundaries
**Mission.** Turn a signed DSSE envelope from the Signer into a **transparencylogged, verifiable fact** with a durable, replayable proof (Merkle inclusion + (optional) checkpoint anchoring). Provide **fast verification** for downstream consumers and a stable retrieval interface for UI/CLI.
**Boundaries.**
* Attestor **does not sign**; it **must not** accept unsigned or thirdpartysigned bundles.
* Attestor **does not decide PASS/FAIL**; it logs attestations for SBOMs, reports, and export artifacts.
* Rekor v2 backends may be **local** (selfhosted) or **remote**; Attestor handles both with retries, backoff, and idempotency.
---
## 1) Topology & dependencies
**Process shape:** single stateless service `stellaops/attestor` behind mTLS.
**Dependencies:**
* **Signer** (caller) — authenticated via **mTLS** and **Authority** OpToks.
* **Rekor v2** — tilebacked transparency log endpoint(s).
* **MinIO (S3)** — optional archive store for DSSE envelopes & verification bundles.
* **MongoDB** — local cache of `{uuid, index, proof, artifactSha256, bundleSha256}`; job state; audit.
* **Redis** — dedupe/idempotency keys and shortlived ratelimit buckets.
* **Licensing Service (optional)** — “endorse” call for crosslog publishing when customer optsin.
Trust boundary: **Only the Signer** is allowed to call submission endpoints; enforced by **mTLS peer cert allowlist** + `aud=attestor` OpTok.
---
## 2) Data model (Mongo)
Database: `attestor`
**Collections & schemas**
* `entries`
```
{ _id: "<rekor-uuid>",
artifact: { sha256: "<sha256>", kind: "sbom|report|vex-export", imageDigest?, subjectUri? },
bundleSha256: "<sha256>", // canonicalized DSSE
index: <int>, // log index/sequence if provided by backend
proof: { // inclusion proof
checkpoint: { origin, size, rootHash, timestamp },
inclusion: { leafHash, path[] } // Merkle path (tiles)
},
log: { url, logId? },
createdAt, status: "included|pending|failed",
signerIdentity: { mode: "keyless|kms", issuer, san?, kid? }
}
```
* `dedupe`
```
{ key: "bundle:<sha256>", rekorUuid, createdAt, ttlAt } // idempotency key
```
* `audit`
```
{ _id, ts, caller: { cn, mTLSThumbprint, sub, aud }, // from mTLS + OpTok
action: "submit|verify|fetch",
artifactSha256, bundleSha256, rekorUuid?, index?, result, latencyMs, backend }
```
Indexes:
* `entries` on `artifact.sha256`, `bundleSha256`, `createdAt`, and `{status:1, createdAt:-1}`.
* `dedupe.key` unique (TTL 2448h).
* `audit.ts` for timerange queries.
---
## 3) Input contract (from Signer)
**Attestor accepts only** DSSE envelopes that satisfy all of:
1. **mTLS** peer certificate maps to `signer` service (CApinned).
2. **Authority** OpTok with `aud=attestor`, `scope=attestor.write`, DPoP or mTLS bound.
3. DSSE envelope is **signed by the Signers key** (or includes a **Fulcioissued** cert chain) and **chains to configured roots** (Fulcio/KMS).
4. **Predicate type** is one of StellaOps types (sbom/report/vexexport) with valid schema.
5. `subject[*].digest.sha256` is present and canonicalized.
**Wire shape (JSON):**
```json
{
"bundle": { "dsse": { "payloadType": "application/vnd.in-toto+json", "payload": "<b64>", "signatures": [ ... ] },
"certificateChain": [ "-----BEGIN CERTIFICATE-----..." ],
"mode": "keyless" },
"meta": {
"artifact": { "sha256": "<subject sha256>", "kind": "sbom|report|vex-export", "imageDigest": "sha256:..." },
"bundleSha256": "<sha256 of canonical dsse>",
"logPreference": "primary", // "primary" | "mirror" | "both"
"archive": true // whether Attestor should archive bundle to S3
}
}
```
---
## 4) APIs
### 4.1 Submission
`POST /api/v1/rekor/entries` *(mTLS + OpTok required)*
* **Body**: as above.
* **Behavior**:
* Verify caller (mTLS + OpTok).
* Validate DSSE bundle (signature, cert chain to Fulcio/KMS; DSSE structure; payloadType allowed).
* Idempotency: compute `bundleSha256`; check `dedupe`. If present, return existing `rekorUuid`.
* Submit canonicalized bundle to Rekor v2 (primary or mirror according to `logPreference`).
* Retrieve **inclusion proof** (blocking until inclusion or up to `proofTimeoutMs`); if backend returns promise only, return `status=pending` and retry asynchronously.
* Persist `entries` record; archive DSSE to S3 if `archive=true`.
* **Response 200**:
```json
{
"uuid": "…",
"index": 123456,
"proof": {
"checkpoint": { "origin": "rekor@site", "size": 987654, "rootHash": "…", "timestamp": "…" },
"inclusion": { "leafHash": "…", "path": ["…","…"] }
},
"logURL": "https://rekor…/api/v2/log/…/entries/…",
"status": "included"
}
```
* **Errors**: `401 invalid_token`, `403 not_signer|chain_untrusted`, `409 duplicate_bundle` (with existing `uuid`), `502 rekor_unavailable`, `504 proof_timeout`.
### 4.2 Proof retrieval
`GET /api/v1/rekor/entries/{uuid}`
* Returns `entries` row (refreshes proof from Rekor if stale/missing).
* Accepts `?refresh=true` to force backend query.
### 4.3 Verification (thirdparty or internal)
`POST /api/v1/rekor/verify`
* **Body** (one of):
* `{ "uuid": "…" }`
* `{ "bundle": { …DSSE… } }`
* `{ "artifactSha256": "…" }` *(looks up most recent entry)*
* **Checks**:
1. **Bundle signature** → cert chain to Fulcio/KMS roots configured.
2. **Inclusion proof** → recompute leaf hash; verify Merkle path against checkpoint root.
3. Optionally verify **checkpoint** against local trust anchors (if Rekor signs checkpoints).
4. Confirm **subject.digest** matches callerprovided hash (when given).
* **Response**:
```json
{ "ok": true, "uuid": "…", "index": 123, "logURL": "…", "checkedAt": "…" }
```
### 4.4 Batch submission (optional)
`POST /api/v1/rekor/batch` accepts an array of submission objects; processes with peritem results.
---
## 5) Rekor v2 driver (backend)
* **Canonicalization**: DSSE envelopes are **normalized** (stable JSON ordering, no insignificant whitespace) before hashing and submission.
* **Transport**: HTTP/2 with retries (exponential backoff, jitter), budgeted timeouts.
* **Idempotency**: if backend returns “already exists,” map to existing `uuid`.
* **Proof acquisition**:
* In synchronous mode, poll the log for inclusion up to `proofTimeoutMs`.
* In asynchronous mode, return `pending` and schedule a **proof fetcher** job (Mongo job doc + backoff).
* **Mirrors/dual logs**:
* When `logPreference="both"`, submit to primary and mirror; store **both** UUIDs (primary canonical).
* Optional **cloud endorsement**: POST to the StellaOps cloud `/attest/endorse` with `{uuid, artifactSha256}`; store returned endorsement id.
---
## 6) Security model
* **mTLS required** for submission from **Signer** (CApinned).
* **Authority token** with `aud=attestor` and DPoP/mTLS binding must be presented; Attestor verifies both.
* **Bundle acceptance policy**:
* DSSE signature must chain to the configured **Fulcio** (keyless) or **KMS/HSM** roots.
* SAN (Subject Alternative Name) must match **Signer identity** policy (e.g., `urn:stellaops:signer` or pinned OIDC issuer).
* Predicate `predicateType` must be on allowlist (sbom/report/vex-export).
* `subject.digest.sha256` values must be present and wellformed (hex).
* **No public submission** path. **Never** accept bundles from untrusted clients.
* **Rate limits**: per mTLS thumbprint/license (from Signerforwarded claims) to avoid flooding the log.
* **Redaction**: Attestor never logs secret material; DSSE payloads **should** be public by design (SBOMs/reports). If customers require redaction, enforce policy at Signer (predicate minimization) **before** Attestor.
---
## 7) Storage & archival
* **Entries** in Mongo provide a local ledger keyed by `rekorUuid` and **artifact sha256** for quick reverse lookups.
* **S3 archival** (if enabled):
```
s3://stellaops/attest/
dsse/<bundleSha256>.json
proof/<rekorUuid>.json
bundle/<artifactSha256>.zip # optional verification bundle
```
* **Verification bundles** (zip):
* DSSE (`*.dsse.json`), proof (`*.proof.json`), `chain.pem` (certs), `README.txt` with verification steps & hashes.
---
## 8) Observability & audit
**Metrics** (Prometheus):
* `attestor.submit_total{result,backend}`
* `attestor.submit_latency_seconds{backend}`
* `attestor.proof_fetch_total{result}`
* `attestor.verify_total{result}`
* `attestor.dedupe_hits_total`
* `attestor.errors_total{type}`
**Tracing**:
* Spans: `validate`, `rekor.submit`, `rekor.poll`, `persist`, `archive`, `verify`.
**Audit**:
* Immutable `audit` rows (ts, caller, action, hashes, uuid, index, backend, result, latency).
---
## 9) Configuration (YAML)
```yaml
attestor:
listen: "https://0.0.0.0:8444"
security:
mtls:
caBundle: /etc/ssl/signer-ca.pem
requireClientCert: true
authority:
issuer: "https://authority.internal"
jwksUrl: "https://authority.internal/jwks"
requireSenderConstraint: "dpop" # or "mtls"
signerIdentity:
mode: ["keyless","kms"]
fulcioRoots: ["/etc/fulcio/root.pem"]
allowedSANs: ["urn:stellaops:signer"]
kmsKeys: ["kms://cluster-kms/stellaops-signer"]
rekor:
primary:
url: "https://rekor-v2.internal"
proofTimeoutMs: 15000
pollIntervalMs: 250
maxAttempts: 60
mirror:
enabled: false
url: "https://rekor-v2.mirror"
mongo:
uri: "mongodb://mongo/attestor"
s3:
enabled: true
endpoint: "http://minio:9000"
bucket: "stellaops"
prefix: "attest/"
objectLock: "governance"
redis:
url: "redis://redis:6379/2"
quotas:
perCaller:
qps: 50
burst: 100
```
---
## 10) Endtoend sequences
**A) Submit & include (happy path)**
```mermaid
sequenceDiagram
autonumber
participant SW as Scanner.WebService
participant SG as Signer
participant AT as Attestor
participant RK as Rekor v2
SW->>SG: POST /sign/dsse (OpTok+PoE)
SG-->>SW: DSSE bundle (+certs)
SW->>AT: POST /rekor/entries (mTLS + OpTok)
AT->>AT: Validate DSSE (chain to Fulcio/KMS; signer identity)
AT->>RK: submit(bundle)
RK-->>AT: {uuid, index?}
AT->>RK: poll inclusion until proof or timeout
RK-->>AT: inclusion proof (checkpoint + path)
AT-->>SW: {uuid, index, proof, logURL}
```
**B) Verify by artifact digest (CLI)**
```mermaid
sequenceDiagram
autonumber
participant CLI as stellaops verify
participant SW as Scanner.WebService
participant AT as Attestor
CLI->>SW: GET /catalog/artifacts/{id}
SW-->>CLI: {artifactSha256, rekor: {uuid}}
CLI->>AT: POST /rekor/verify { uuid }
AT-->>CLI: { ok: true, index, logURL }
```
---
## 11) Failure modes & responses
| Condition | Return | Details | | |
| ------------------------------------- | ----------------------- | --------------------------------------------------------- | -------- | ------------ |
| mTLS/OpTok invalid | `401 invalid_token` | Include `WWW-Authenticate` DPoP challenge when applicable | | |
| Bundle not signed by trusted identity | `403 chain_untrusted` | DSSE accepted only from Signer identities | | |
| Duplicate bundle | `409 duplicate_bundle` | Return existing `uuid` (idempotent) | | |
| Rekor unreachable/timeout | `502 rekor_unavailable` | Retry with backoff; surface `Retry-After` | | |
| Inclusion proof timeout | `202 accepted` | `status=pending`, background job continues to fetch proof | | |
| Archive failure | `207 multi-status` | Entry recorded; archive will retry asynchronously | | |
| Verification mismatch | `400 verify_failed` | Include reason: chain | leafHash | rootMismatch |
---
## 12) Performance & scale
* Stateless; scale horizontally.
* **Targets**:
* Submit+proof P95 ≤ **300ms** (warm log; local Rekor).
* Verify P95 ≤ **30ms** from cache; ≤ **120ms** with live proof fetch.
* 1k submissions/minute per replica sustained.
* **Hot caches**: `dedupe` (bundle hash → uuid), recent `entries` by artifact sha256.
---
## 13) Testing matrix
* **Happy path**: valid DSSE, inclusion within timeout.
* **Idempotency**: resubmit same `bundleSha256` → same `uuid`.
* **Security**: reject nonSigner mTLS, wrong `aud`, DPoP replay, untrusted cert chain, forbidden predicateType.
* **Rekor variants**: promisethenproof, proof delayed, mirror dualsubmit, mirror failure.
* **Verification**: corrupt leaf path, wrong root, tampered bundle.
* **Throughput**: soak test with 10k submissions; latency SLOs, zero drops.
---
## 14) Implementation notes
* Language: **.NET 10** minimal API; `HttpClient` with **sockets handler** tuned for HTTP/2.
* JSON: **canonical writer** for DSSE payload hashing.
* Crypto: use **BouncyCastle**/**System.Security.Cryptography**; PEM parsing for cert chains.
* Rekor client: pluggable driver; treat backend errors as retryable/nonretryable with granular mapping.
* Safety: size caps on bundles; decompress bombs guarded; strict UTF8.
* CLI integration: `stellaops verify attestation <uuid|bundle|artifact>` calls `/rekor/verify`.
---
## 15) Optional features
* **Duallog** write (primary + mirror) and **crosslog proof** packaging.
* **Cloud endorsement**: send `{uuid, artifactSha256}` to StellaOps cloud; store returned endorsement id for marketing/chainofcustody.
* **Checkpoint pinning**: periodically pin latest Rekor checkpoints to an external audit store for independent monitoring.

View File

@@ -0,0 +1,394 @@
# component_architecture_authority.md — **StellaOps Authority** (2025Q4)
> **Scope.** Implementationready architecture for **StellaOps Authority**: the onprem **OIDC/OAuth2** service that issues **shortlived, senderconstrained operational tokens (OpToks)** to firstparty services and tools. Covers protocols (DPoP & mTLS binding), token shapes, endpoints, storage, rotation, HA, RBAC, audit, and testing. This component is the trust anchor for *who* is calling inside a StellaOps installation. (Entitlement is proven separately by **PoE** from the cloud Licensing Service; Authority does not issue PoE.)
---
## 0) Mission & boundaries
**Mission.** Provide **fast, local, verifiable** authentication for StellaOps microservices and tools by minting **very shortlived** OAuth2/OIDC tokens that are **senderconstrained** (DPoP or mTLSbound). Support RBAC scopes, multitenant claims, and deterministic validation for APIs (Scanner, Signer, Attestor, Excititor, Concelier, UI, CLI, Zastava).
**Boundaries.**
* Authority **does not** validate entitlements/licensing. Thats enforced by **Signer** using **PoE** with the cloud Licensing Service.
* Authority tokens are **operational only** (25min TTL) and must not be embedded in longlived artifacts or stored in SBOMs.
* Authority is **stateless for validation** (JWT) and **optional introspection** for services that prefer online checks.
---
## 1) Protocols & cryptography
* **OIDC Discovery**: `/.well-known/openid-configuration`
* **OAuth2** grant types:
* **Client Credentials** (service↔service, with mTLS or private_key_jwt)
* **Device Code** (CLI login on headless agents; optional)
* **Authorization Code + PKCE** (browser login for UI; optional)
* **Sender constraint options** (choose per caller or per audience):
* **DPoP** (Demonstration of ProofofPossession): proof JWT on each HTTP request, bound to the access token via `cnf.jkt`.
* **OAuth 2.0 mTLS** (certificatebound tokens): token bound to client certificate thumbprint via `cnf.x5t#S256`.
* **Signing algorithms**: **EdDSA (Ed25519)** preferred; fallback **ES256 (P256)**. Rotation is supported via **kid** in JWKS.
* **Token format**: **JWT** access tokens (compact), optionally opaque reference tokens for services that insist on introspection.
* **Clock skew tolerance**: ±60s; issue `nbf`, `iat`, `exp` accordingly.
---
## 2) Token model
### 2.1 Access token (OpTok) — shortlived (120300s)
**Registered claims**
```
iss = https://authority.<domain>
sub = <client_id or user_id>
aud = <service audience: signer|scanner|attestor|concelier|excititor|ui|zastava>
exp = <unix ts> (<= 300 s from iat)
iat = <unix ts>
nbf = iat - 30
jti = <uuid>
scope = "scanner.scan scanner.export signer.sign ..."
```
**Senderconstraint (`cnf`)**
* **DPoP**:
```json
"cnf": { "jkt": "<base64url(SHA-256(JWK))>" }
```
* **mTLS**:
```json
"cnf": { "x5t#S256": "<base64url(SHA-256(client_cert_der))>" }
```
**Install/tenant context (custom claims)**
```
tid = <tenant id> // multi-tenant
inst = <installation id> // unique installation
roles = [ "svc.scanner", "svc.signer", "ui.admin", ... ]
plan? = <plan name> // optional hint for UIs; not used for enforcement
```
> **Note**: Do **not** copy PoE claims into OpTok; OpTok ≠ entitlement. Only **Signer** checks PoE.
### 2.2 Refresh tokens (optional)
* Default **disabled**. If enabled (for UI interactive logins), pair with **DPoPbound** refresh tokens or **mTLS** client sessions; short TTL (≤ 8h), rotating on use (replaysafe).
### 2.3 ID tokens (optional)
* Issued for UI/browser OIDC flows (Authorization Code + PKCE); not used for service auth.
---
## 3) Endpoints & flows
### 3.1 OIDC discovery & keys
* `GET /.well-known/openid-configuration` → endpoints, algs, jwks_uri
* `GET /jwks` → JSON Web Key Set (rotating, at least 2 active keys during transition)
### 3.2 Token issuance
* `POST /oauth/token`
* **Client Credentials** (service→service):
* **mTLS**: mutual TLS + `client_id` → bound token (`cnf.x5t#S256`)
* **private_key_jwt**: JWTbased client auth + **DPoP** header (preferred for tools and CLI)
* **Device Code** (CLI): `POST /oauth/device/code` + `POST /oauth/token` poll
* **Authorization Code + PKCE** (UI): standard
**DPoP handshake (example)**
1. Client prepares **JWK** (ephemeral keypair).
2. Client sends **DPoP proof** header with fields:
```
htm=POST
htu=https://authority.../oauth/token
iat=<now>
jti=<uuid>
```
signed with the DPoP private key; header carries JWK.
3. Authority validates proof; issues access token with `cnf.jkt=<thumbprint(JWK)>`.
4. Client uses the same DPoP key to sign **every subsequent API request** to services (Signer, Scanner, …).
**mTLS flow**
* Mutual TLS at the connection; Authority extracts client cert, validates chain; token carries `cnf.x5t#S256`.
### 3.3 Introspection & revocation (optional)
* `POST /oauth/introspect` → `{ active, sub, scope, aud, exp, cnf, ... }`
* `POST /oauth/revoke` → revokes refresh tokens or opaque access tokens.
* **Replay prevention**: maintain **DPoP `jti` cache** (TTL ≤ 10 min) to reject duplicate proofs when services supply DPoP nonces (Signer requires nonce for highvalue operations).
### 3.4 UserInfo (optional for UI)
* `GET /userinfo` (ID token context).
---
## 4) Audiences, scopes & RBAC
### 4.1 Audiences
* `signer` — only the **Signer** service should accept tokens with `aud=signer`.
* `attestor`, `scanner`, `concelier`, `excititor`, `ui`, `zastava` similarly.
Services **must** verify `aud` and **sender constraint** (DPoP/mTLS) per their policy.
### 4.2 Core scopes
| Scope | Service | Operation |
| ---------------------------------- | ------------------ | -------------------------- |
| `signer.sign` | Signer | Request DSSE signing |
| `attestor.write` | Attestor | Submit Rekor entries |
| `scanner.scan` | Scanner.WebService | Submit scan jobs |
| `scanner.export` | Scanner.WebService | Export SBOMs |
| `scanner.read` | Scanner.WebService | Read catalog/SBOMs |
| `vex.read` / `vex.admin` | Excititor | Query/operate |
| `concelier.read` / `concelier.export` | Concelier | Query/exports |
| `ui.read` / `ui.admin` | UI | View/admin |
| `zastava.emit` / `zastava.enforce` | Scanner/Zastava | Runtime events / admission |
**Roles → scopes mapping** is configured centrally (Authority policy) and pushed during token issuance.
---
## 5) Storage & state
* **Configuration DB** (PostgreSQL/MySQL): clients, audiences, role→scope maps, tenant/installation registry, device code grants, persistent consents (if any).
* **Cache** (Redis):
* DPoP **jti** replay cache (short TTL)
* **Nonce** store (per resource server, if they demand nonce)
* Device code pollers, rate limiting buckets
* **JWKS**: key material in HSM/KMS or encrypted at rest; JWKS served from memory.
---
## 6) Key management & rotation
* Maintain **at least 2 signing keys** active during rotation; tokens carry `kid`.
* Prefer **Ed25519** for compact tokens; maintain **ES256** fallback for FIPS contexts.
* Rotation cadence: 3090 days; emergency rotation supported.
* Publish new JWKS **before** issuing tokens with the new `kid` to avoid coldstart validation misses.
* Keep **old keys** available **at least** for max token TTL + 5 minutes.
---
## 7) HA & performance
* **Stateless issuance** (except device codes/refresh) → scale horizontally behind a loadbalancer.
* **DB** only for client metadata and optional flows; token checks are JWTlocal; introspection endpoints hit cache/DB minimally.
* **Targets**:
* Token issuance P95 ≤ **20ms** under warm cache.
* DPoP proof validation ≤ **1ms** extra per request at resource servers (Signer/Scanner).
* 99.9% uptime; HPA on CPU/latency.
---
## 8) Security posture
* **Strict TLS** (1.3 preferred); HSTS; modern cipher suites.
* **mTLS** enabled where required (Signer/Attestor paths).
* **Replay protection**: DPoP `jti` cache, nonce support for **Signer** (add `DPoP-Nonce` header on 401; clients resign).
* **Rate limits** per client & per IP; exponential backoff on failures.
* **Secrets**: clients use **private_key_jwt** or **mTLS**; never basic secrets over the wire.
* **CSP/CSRF** hardening on UI flows; `SameSite=Lax` cookies; PKCE enforced.
* **Logs** redact `Authorization` and DPoP proofs; store `sub`, `aud`, `scopes`, `inst`, `tid`, `cnf` thumbprints, not full keys.
---
## 9) Multitenancy & installations
* **Tenant (`tid`)** and **Installation (`inst`)** registries define which audiences/scopes a client can request.
* Crosstenant isolation enforced at issuance (disallow rogue `aud`), and resource servers **must** check that `tid` matches their configured tenant.
---
## 10) Admin & operations APIs
All under `/admin` (mTLS + `authority.admin` scope).
```
POST /admin/clients # create/update client (confidential/public)
POST /admin/audiences # register audience resource URIs
POST /admin/roles # define role→scope mappings
POST /admin/tenants # create tenant/install entries
POST /admin/keys/rotate # rotate signing key (zero-downtime)
GET /admin/metrics # Prometheus exposition (token issue rates, errors)
GET /admin/healthz|readyz # health/readiness
```
---
## 11) Integration hard lines (what resource servers must enforce)
Every StellaOps service that consumes Authority tokens **must**:
1. Verify JWT signature (`kid` in JWKS), `iss`, `aud`, `exp`, `nbf`.
2. Enforce **senderconstraint**:
* **DPoP**: validate DPoP proof (`htu`, `htm`, `iat`, `jti`) and match `cnf.jkt`; cache `jti` for replay defense; honor nonce challenges.
* **mTLS**: match presented client cert thumbprint to token `cnf.x5t#S256`.
3. Check **scopes**; optionally map to internal roles.
4. Check **tenant** (`tid`) and **installation** (`inst`) as appropriate.
5. For **Signer** only: require **both** OpTok and **PoE** in the request (enforced by Signer, not Authority).
---
## 12) Error surfaces & UX
* Token endpoint errors follow OAuth2 (`invalid_client`, `invalid_grant`, `invalid_scope`, `unauthorized_client`).
* Resource servers use RFC6750 style (`WWW-Authenticate: DPoP error="invalid_token", error_description="…", dpop_nonce="…" `).
* For DPoP nonce challenges, clients retry with the serversupplied nonce once.
---
## 13) Observability & audit
* **Metrics**:
* `authority.tokens_issued_total{grant,aud}`
* `authority.dpop_validations_total{result}`
* `authority.mtls_bindings_total{result}`
* `authority.jwks_rotations_total`
* `authority.errors_total{type}`
* **Audit log** (immutable sink): token issuance (`sub`, `aud`, `scopes`, `tid`, `inst`, `cnf thumbprint`, `jti`), revocations, admin changes.
* **Tracing**: token flows, DB reads, JWKS cache.
---
## 14) Configuration (YAML)
```yaml
authority:
issuer: "https://authority.internal"
keys:
algs: [ "EdDSA", "ES256" ]
rotationDays: 60
storage: kms://cluster-kms/authority-signing
tokens:
accessTtlSeconds: 180
enableRefreshTokens: false
clockSkewSeconds: 60
dpop:
enable: true
nonce:
enable: true
ttlSeconds: 600
mtls:
enable: true
caBundleFile: /etc/ssl/mtls/clients-ca.pem
clients:
- clientId: scanner-web
grantTypes: [ "client_credentials" ]
audiences: [ "scanner" ]
auth: { type: "private_key_jwt", jwkFile: "/secrets/scanner-web.jwk" }
senderConstraint: "dpop"
scopes: [ "scanner.scan", "scanner.export", "scanner.read" ]
- clientId: signer
grantTypes: [ "client_credentials" ]
audiences: [ "signer" ]
auth: { type: "mtls" }
senderConstraint: "mtls"
scopes: [ "signer.sign" ]
```
---
## 15) Testing matrix
* **JWT validation**: wrong `aud`, expired `exp`, skewed `nbf`, stale `kid`.
* **DPoP**: invalid `htu`/`htm`, replayed `jti`, stale `iat`, wrong `jkt`, nonce dance.
* **mTLS**: wrong client cert, wrong CA, thumbprint mismatch.
* **RBAC**: scope enforcement per audience; overprivileged client denied.
* **Rotation**: JWKS rotation while loadtesting; zerodowntime verification.
* **HA**: kill one Authority instance; verify issuance continues; JWKS served by peers.
* **Performance**: 1k token issuance/sec on 2 cores with Redis enabled for jti caching.
---
## 16) Threat model & mitigations (summary)
| Threat | Vector | Mitigation |
| ------------------- | ---------------- | ------------------------------------------------------------------------------------------ |
| Token theft | Copy of JWT | **Short TTL**, **senderconstraint** (DPoP/mTLS); replay blocked by `jti` cache and nonces |
| Replay across hosts | Reuse DPoP proof | Enforce `htu`/`htm`, `iat` freshness, `jti` uniqueness; services may require **nonce** |
| Impersonation | Fake client | mTLS or `private_key_jwt` with pinned JWK; client registration & rotation |
| Key compromise | Signing key leak | HSM/KMS storage, key rotation, audit; emergency key revoke path; narrow token TTL |
| Crosstenant abuse | Scope elevation | Enforce `aud`, `tid`, `inst` at issuance and resource servers |
| Downgrade to bearer | Strip DPoP | Resource servers require DPoP/mTLS based on `aud`; reject bearer without `cnf` |
---
## 17) Deployment & HA
* **Stateless** microservice, containerized; run ≥ 2 replicas behind LB.
* **DB**: HA Postgres (or MySQL) for clients/roles; **Redis** for device codes, DPoP nonces/jtis.
* **Secrets**: mount client JWKs via K8s Secrets/HashiCorp Vault; signing keys via KMS.
* **Backups**: DB daily; Redis not critical (ephemeral).
* **Disaster recovery**: export/import of client registry; JWKS rehydrate from KMS.
* **Compliance**: TLS audit; penetration testing for OIDC flows.
---
## 18) Implementation notes
* Reference stack: **.NET 10** + **OpenIddict 6** (or IdentityServer if licensed) with custom DPoP validator and mTLS binding middleware.
* Keep the DPoP/JTI cache pluggable; allow Redis/Memcached.
* Provide **client SDKs** for C# and Go: DPoP key mgmt, proof generation, nonce handling, token refresh helper.
---
## 19) Quick reference — wire examples
**Access token (payload excerpt)**
```json
{
"iss": "https://authority.internal",
"sub": "scanner-web",
"aud": "signer",
"exp": 1760668800,
"iat": 1760668620,
"nbf": 1760668620,
"jti": "9d9c3f01-6e1a-49f1-8f77-9b7e6f7e3c50",
"scope": "signer.sign",
"tid": "tenant-01",
"inst": "install-7A2B",
"cnf": { "jkt": "KcVb2V...base64url..." }
}
```
**DPoP proof header fields (for POST /sign/dsse)**
```json
{
"htu": "https://signer.internal/sign/dsse",
"htm": "POST",
"iat": 1760668620,
"jti": "4b1c9b3c-8a95-4c58-8a92-9c6cfb4a6a0b"
}
```
Signer validates that `hash(JWK)` in the proof matches `cnf.jkt` in the token.
---
## 20) Rollout plan
1. **MVP**: Client Credentials (private_key_jwt + DPoP), JWKS, short OpToks, peraudience scopes.
2. **Add**: mTLSbound tokens for Signer/Attestor; device code for CLI; optional introspection.
3. **Hardening**: DPoP nonce support; full audit pipeline; HA tuning.
4. **UX**: Tenant/installation admin UI; role→scope editors; client bootstrap wizards.

389
docs/ARCHITECTURE_CLI.md Normal file
View File

@@ -0,0 +1,389 @@
# component_architecture_cli.md — **StellaOps CLI** (2025Q4)
> **Scope.** Implementationready architecture for **StellaOps CLI**: command surface, process model, auth (Authority/DPoP), integration with Scanner/Excititor/Concelier/Signer/Attestor, Buildx plugin management, offline kit behavior, packaging, observability, security posture, and CI ergonomics.
---
## 0) Mission & boundaries
**Mission.** Provide a **fast, deterministic, CIfriendly** commandline interface to drive StellaOps workflows:
* Buildtime SBOM generation via **Buildx generator** orchestration.
* Postbuild **scan/compose/diff/export** against **Scanner.WebService**.
* **Policy** operations and **VEX/Vuln** data pulls (operator tasks).
* **Verification** (attestation, referrers, signatures) for audits.
* Airgapped/offline **kit** administration.
**Boundaries.**
* CLI **never** signs; it only calls **Signer**/**Attestor** via backend APIs when needed (e.g., `report --attest`).
* CLI **does not** store longlived credentials beyond OS keychain; tokens are **short** (Authority OpToks).
* Heavy work (scanning, merging, policy) is executed **serverside** (Scanner/Excititor/Concelier).
---
## 1) Solution layout & runtime form
```
src/
├─ StellaOps.Cli/ # net10.0 (Native AOT) single binary
├─ StellaOps.Cli.Core/ # verb plumbing, config, HTTP, auth
├─ StellaOps.Cli.Plugins/ # optional verbs packaged as plugins
├─ StellaOps.Cli.Tests/ # unit + golden-output tests
└─ packaging/
├─ msix / msi / deb / rpm / brew formula
└─ scoop manifest / winget manifest
```
**Language/runtime**: .NET 10 **Native AOT** for speed/startup; Linux builds use **musl** static when possible.
**OS targets**: linuxx64/arm64, windowsx64/arm64, macOSx64/arm64.
---
## 2) Command surface (verbs)
> All verbs default to **JSON** output when `--json` is set (CI mode). Human output is concise, deterministic.
### 2.1 Auth & profile
* `auth login`
* Modes: **devicecode** (default), **clientcredentials** (service principal).
* Produces **Authority** access token (OpTok) + stores **DPoP** keypair in OS keychain.
* `auth status` — show current issuer, subject, audiences, expiry.
* `auth logout` — wipe cached tokens/keys.
### 2.2 Buildtime SBOM (Buildx)
* `buildx install` — install/update the **StellaOps.Scanner.Sbomer.BuildXPlugin** on the host.
* `buildx verify` — ensure generator is usable.
* `buildx build` — thin wrapper around `docker buildx build --attest=type=sbom,generator=stellaops/sbom-indexer` with convenience flags:
* `--attest` (request Signer/Attestor via backend postpush)
* `--provenance` passthrough (optional)
### 2.3 Scanning & artifacts
* `scan image <ref|digest>`
* Options: `--force`, `--wait`, `--view=inventory|usage|both`, `--format=cdx-json|cdx-pb|spdx-json`, `--attest` (ask backend to sign/log).
* Streams progress; exits early unless `--wait`.
* `diff image --old <digest> --new <digest> [--view ...]` — show layerattributed changes.
* `export sbom <digest> [--view ... --format ... --out file]` — download artifact.
* `report final <digest> [--policy-revision ... --attest]` — request PASS/FAIL report from backend (policy+vex) and optional attestation.
### 2.4 Policy & data
* `policy get/set/apply` — fetch active policy, apply staged policy, compute digest.
* `concelier export` — trigger/export canonical JSON or Trivy DB (admin).
* `excititor export` — trigger/export consensus/raw claims (admin).
### 2.5 Verification
* `verify attestation --uuid <rekor-uuid> | --artifact <sha256> | --bundle <path>` — call **Attestor /verify** and print proof summary.
* `verify referrers <digest>` — ask **Signer /verify/referrers** (is image Stellasigned?).
* `verify image-signature <ref|digest>` — standalone cosign verification (optional, local).
### 2.6 Runtime (Zastava helper)
* `runtime policy test --images <digest,...> [--ns <name> --labels k=v,...]` — ask backend `/policy/runtime` like the webhook would.
### 2.7 Offline kit
* `offline kit pull` — fetch latest **Concelier JSON + Trivy DB + Excititor exports** as a tarball from a mirror.
* `offline kit import <tar>` — upload the kit to onprem services (Concelier/Excititor).
* `offline kit status` — list current seed versions.
### 2.8 Utilities
* `config set/get` — endpoint & defaults.
* `whoami` — short auth display.
* `version` — CLI + protocol versions; release channel.
---
## 3) AuthN: Authority + DPoP
### 3.1 Token acquisition
* **Devicecode**: the CLI opens an OIDC device code flow against **Authority**; the browser login is optional for service principals.
* **Clientcredentials**: service principals use **private_key_jwt** or **mTLS** to get tokens.
### 3.2 DPoP key management
* On first login, the CLI generates an **ephemeral JWK** (Ed25519) and stores it in the **OS keychain** (Keychain/DPAPI/KWallet/Gnome Keyring).
* Every request to backend services includes a **DPoP proof**; CLI refreshes tokens as needed.
### 3.3 Multiaudience & scopes
* CLI requests **audiences** as needed per verb:
* `scanner` for scan/export/report/diff
* `signer` (indirect; usually backend calls Signer)
* `attestor` for verify
* `concelier`/`excititor` for admin verbs
CLI rejects verbs if required scopes are missing.
---
## 4) Process model & reliability
### 4.1 HTTP client
* Single **http2** client with connection pooling, DNS pinning, retry/backoff (idempotent GET/POST marked safe).
* **DPoP nonce** handling: on `401` with nonce challenge, CLI replays once.
### 4.2 Streaming
* `scan` and `report` support **serversent JSON lines** (progress events).
* `--json` prints machine events; human mode shows compact spinners and crucial updates only.
### 4.3 Exit codes (CIsafe)
| Code | Meaning |
| ---- | ------------------------------------------- |
| 0 | Success |
| 2 | Policy fail (final report verdict=fail) |
| 3 | Verification failed (attestation/signature) |
| 4 | Auth error (invalid/missing token/DPoP) |
| 5 | Resource not found (image/SBOM) |
| 6 | Rate limited / quota exceeded |
| 7 | Backend unavailable (retryable) |
| 9 | Invalid arguments |
---
## 5) Configuration model
**Precedence:** CLI flags → env vars → config file → defaults.
**Config file**: `${XDG_CONFIG_HOME}/stellaops/config.yaml` (Windows: `%APPDATA%\StellaOps\config.yaml`)
```yaml
cli:
authority: "https://authority.internal"
backend:
scanner: "https://scanner-web.internal"
attestor: "https://attestor.internal"
concelier: "https://concelier-web.internal"
excititor: "https://excititor-web.internal"
auth:
audienceDefault: "scanner"
deviceCode: true
output:
json: false
color: auto
tls:
caBundle: "/etc/ssl/certs/ca-bundle.crt"
offline:
kitMirror: "s3://mirror/stellaops-kit"
```
Environment variables: `STELLAOPS_AUTHORITY`, `STELLAOPS_SCANNER_URL`, etc.
---
## 6) Buildx generator orchestration
* `buildx install` locates the Docker root directory, writes the **generator** plugin manifest, and pulls `stellaops/sbom-indexer` image (pinned digest).
* `buildx build` wrapper injects:
* `--attest=type=sbom,generator=stellaops/sbom-indexer`
* `--label org.stellaops.request=sbom`
* Postbuild: CLI optionally calls **Scanner.WebService** to **verify referrers**, **compose** image SBOMs, and **attest** via Signer/Attestor.
**Detection**: If Buildx or generator unavailable, CLI falls back to **postbuild scan** with a warning.
---
## 7) Artifact handling
* **Downloads** (`export sbom`, `report final`): stream to file; compute sha256 on the fly; write sidecar `.sha256` and optional **verification bundle** (if `--bundle`).
* **Uploads** (`offline kit import`): chunked upload; retry on transient errors; show progress bar (unless `--json`).
---
## 8) Security posture
* **DPoP private keys** stored in **OS keychain**; metadata cached in config.
* **No plaintext tokens** on disk; shortlived **OpToks** held in memory.
* **TLS**: verify backend certificates; allow custom CA bundle for onprem.
* **Redaction**: CLI logs remove `Authorization`, DPoP headers, PoE tokens.
* **Supply chain**: CLI distribution binaries are **cosignsigned**; `stellaops version --verify` checks its own signature.
---
## 9) Observability
* `--verbose` adds request IDs, timings, and retry traces.
* **Metrics** (optional, disabled by default): Prometheus text file exporter for local monitoring in longrunning agents.
* **Structured logs** (`--json`): perevent JSON lines with `ts`, `verb`, `status`, `latencyMs`.
---
## 10) Performance targets
* Startup ≤ **20ms** (AOT).
* `scan image` request/response overhead ≤ **5ms** (excluding server work).
* Buildx wrapper overhead negligible (<1ms).
* Large artifact download (100MB) sustained **80MB/s** on local networks.
---
## 11) Tests & golden outputs
* **Unit tests**: argument parsing, config precedence, URL resolution, DPoP proof creation.
* **Integration tests** (Testcontainers): mock Authority/Scanner/Attestor; CI pipeline with fake registry.
* **Golden outputs**: verb snapshots for `--json` across OSes; kept in `tests/golden/…`.
* **Contract tests**: ensure API shapes match service OpenAPI; fail build if incompatible.
---
## 12) Error envelopes (human + JSON)
**Human:**
```
✖ Policy FAIL: 3 high, 1 critical (VEX suppressed 12)
- pkg:rpm/openssl (CVE-2025-12345) — affected (vendor) — fixed in 3.0.14
- pkg:npm/lodash (GHSA-xxxx) — affected — no fix
See: https://ui.internal/scans/sha256:...
Exit code: 2
```
**JSON (`--json`):**
```json
{ "event":"report", "status":"fail", "critical":1, "high":3, "url":"https://ui..." }
```
---
## 13) Admin & advanced flags
* `--authority`, `--scanner`, `--attestor`, `--concelier`, `--excititor` override config URLs.
* `--no-color`, `--quiet`, `--json`.
* `--timeout`, `--retries`, `--retry-backoff-ms`.
* `--ca-bundle`, `--insecure` (dev only; prints warning).
* `--trace` (dump HTTP traces to file; scrubbed).
---
## 14) Interop with other tools
* Emits **CycloneDX Protobuf** directly to stdout when `export sbom --format cdx-pb --out -`.
* Pipes to `jq`/`yq` cleanly in JSON mode.
* Can act as a **credential helper** for scripts: `stellaops auth token --aud scanner` prints a oneshot token for curl.
---
## 15) Packaging & distribution
* **Installers**: deb/rpm (postinst registers completions), Homebrew, Scoop, Winget, MSI/MSIX.
* **Shell completions**: bash/zsh/fish/pwsh.
* **Update channel**: `stellaops self-update` (optional) fetches cosignsigned release manifest; corporate environments can disable.
---
## 16) Security hard lines
* Refuse to print token values; redact Authorization headers in verbose output.
* Disallow `--insecure` unless `STELLAOPS_CLI_ALLOW_INSECURE=1` set (double optin).
* Enforce **short token TTL**; refresh proactively when <30s left.
* Devicecode cache binding to **machine** and **user** (protect against copy to other machines).
---
## 17) Wire sequences
**A) Scan & wait with attestation**
```mermaid
sequenceDiagram
autonumber
participant CLI
participant Auth as Authority
participant SW as Scanner.WebService
participant SG as Signer
participant AT as Attestor
CLI->>Auth: device code flow (DPoP)
Auth-->>CLI: OpTok (aud=scanner)
CLI->>SW: POST /scans { imageRef, attest:true }
SW-->>CLI: { scanId }
CLI->>SW: GET /scans/{id} (poll)
SW-->>CLI: { status: completed, artifacts, rekor? } # if attested
alt attestation pending
SW->>SG: POST /sign/dsse (server-side)
SG-->>SW: DSSE
SW->>AT: POST /rekor/entries
AT-->>SW: { uuid, proof }
end
CLI->>SW: GET /sboms/<digest>?format=cdx-pb&view=usage
SW-->>CLI: bytes
```
**B) Verify attestation by artifact**
```mermaid
sequenceDiagram
autonumber
participant CLI
participant AT as Attestor
CLI->>AT: POST /rekor/verify { artifactSha256 }
AT-->>CLI: { ok:true, uuid, index, logURL }
```
---
## 18) Roadmap (CLI)
* `scan fs <path>` (local filesystem tree) upload to backend for analysis.
* `policy test --sbom <file>` (simulate policy results offline using local policy bundle).
* `runtime capture` (developer mode) capture small `/proc/<pid>/maps` for troubleshooting.
* Pluggable output renderers for SARIF/HTML (admincontrolled).
---
## 19) Example CI snippets
**GitHub Actions (postbuild)**
```yaml
- name: Login (device code w/ OIDC broker)
run: stellaops auth login --json --authority ${{ secrets.AUTHORITY_URL }}
- name: Scan
run: stellaops scan image ${{ steps.build.outputs.digest }} --wait --json
- name: Export (usage view, protobuf)
run: stellaops export sbom ${{ steps.build.outputs.digest }} --view usage --format cdx-pb --out sbom.pb
- name: Verify attestation
run: stellaops verify attestation --artifact $(sha256sum sbom.pb | cut -d' ' -f1) --json
```
**GitLab (buildx generator)**
```yaml
script:
- stellaops buildx install
- docker buildx build --attest=type=sbom,generator=stellaops/sbom-indexer -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA .
- stellaops scan image $CI_REGISTRY_IMAGE@$IMAGE_DIGEST --wait --json
```
---
## 20) Test matrix (OS/arch)
* Linux: ubuntu20.04/22.04/24.04 (x64, arm64), alpine (musl).
* macOS: 1315 (x64, arm64).
* Windows: 10/11, Server 2019/2022 (x64, arm64).
* Docker engines: Docker Desktop, containerdbased runners.

View File

@@ -0,0 +1,433 @@
# component_architecture_concelier.md — **StellaOps Concelier** (2025Q4)
> **Scope.** Implementationready architecture for **Concelier**: the vulnerability ingest/normalize/merge/export subsystem that produces deterministic advisory data for the Scanner + Policy + Excititor pipeline. Covers domain model, connectors, merge rules, storage schema, exports, APIs, performance, security, and test matrices.
---
## 0) Mission & boundaries
**Mission.** Acquire authoritative **vulnerability advisories** (vendor PSIRTs, distros, OSS ecosystems, CERTs), normalize them into a **canonical model**, reconcile aliases and version ranges, and export **deterministic artifacts** (JSON, Trivy DB) for fast backend joins.
**Boundaries.**
* Concelier **does not** sign with private keys. When attestation is required, the export artifact is handed to the **Signer**/**Attestor** pipeline (outofprocess).
* Concelier **does not** decide PASS/FAIL; it provides data to the **Policy** engine.
* Online operation is **allowlistonly**; airgapped deployments use the **Offline Kit**.
---
## 1) Topology & processes
**Process shape:** single ASP.NET Core service `StellaOps.Concelier.WebService` hosting:
* **Scheduler** with distributed locks (Mongo backed).
* **Connectors** (fetch/parse/map).
* **Merger** (canonical record assembly + precedence).
* **Exporters** (JSON, Trivy DB).
* **Minimal REST** for health/status/trigger/export.
**Scale:** HA by running N replicas; **locks** prevent overlapping jobs per source/exporter.
---
## 2) Canonical domain model
> Stored in MongoDB (database `concelier`), serialized with a **canonical JSON** writer (stable order, camelCase, normalized timestamps).
### 2.1 Core entities
**Advisory**
```
advisoryId // internal GUID
advisoryKey // stable string key (e.g., CVE-2025-12345 or vendor ID)
title // short title (best-of from sources)
summary // normalized summary (English; i18n optional)
published // earliest source timestamp
modified // latest source timestamp
severity // normalized {none, low, medium, high, critical}
cvss // {v2?, v3?, v4?} objects (vector, baseScore, severity, source)
exploitKnown // bool (e.g., KEV/active exploitation flags)
references[] // typed links (advisory, kb, patch, vendor, exploit, blog)
sources[] // provenance for traceability (doc digests, URIs)
```
**Alias**
```
advisoryId
scheme // CVE, GHSA, RHSA, DSA, USN, MSRC, etc.
value // e.g., "CVE-2025-12345"
```
**Affected**
```
advisoryId
productKey // canonical product identity (see 2.2)
rangeKind // semver | evr | nvra | apk | rpm | deb | generic | exact
introduced? // string (format depends on rangeKind)
fixed? // string (format depends on rangeKind)
lastKnownSafe? // optional explicit safe floor
arch? // arch or platform qualifier if source declares (x86_64, aarch64)
distro? // distro qualifier when applicable (rhel:9, debian:12, alpine:3.19)
ecosystem? // npm|pypi|maven|nuget|golang|…
notes? // normalized notes per source
```
**Reference**
```
advisoryId
url
kind // advisory | patch | kb | exploit | mitigation | blog | cvrf | csaf
sourceTag // e.g., vendor/redhat, distro/debian, oss/ghsa
```
**MergeEvent**
```
advisoryKey
beforeHash // canonical JSON hash before merge
afterHash // canonical JSON hash after merge
mergedAt
inputs[] // source doc digests that contributed
```
**ExportState**
```
exportKind // json | trivydb
baseExportId? // last full baseline
baseDigest? // digest of last full baseline
lastFullDigest? // digest of last full export
lastDeltaDigest? // digest of last delta export
cursor // per-kind incremental cursor
files[] // last manifest snapshot (path → sha256)
```
### 2.2 Product identity (`productKey`)
* **Primary:** `purl` (Package URL).
* **OS packages:** RPM (NEVRA→purl:rpm), DEB (dpkg→purl:deb), APK (apk→purl:alpine), with **EVR/NVRA** preserved.
* **Secondary:** `cpe` retained for compatibility; advisory records may carry both.
* **Image/platform:** `oci:<registry>/<repo>@<digest>` for imagelevel advisories (rare).
* **Unmappable:** if a source is nondeterministic, keep native string under `productKey="native:<provider>:<id>"` and mark **nonjoinable**.
---
## 3) Source families & precedence
### 3.1 Families
* **Vendor PSIRTs**: Microsoft, Oracle, Cisco, Adobe, Apple, VMware, Chromium…
* **Linux distros**: Red Hat, SUSE, Ubuntu, Debian, Alpine…
* **OSS ecosystems**: OSV, GHSA (GitHub Security Advisories), PyPI, npm, Maven, NuGet, Go.
* **CERTs / national CSIRTs**: CISA (KEV, ICS), JVN, ACSC, CCCS, KISA, CERTFR/BUND, etc.
### 3.2 Precedence (when claims conflict)
1. **Vendor PSIRT** (authoritative for their product).
2. **Distro** (authoritative for packages they ship, including backports).
3. **Ecosystem** (OSV/GHSA) for library semantics.
4. **CERTs/aggregators** for enrichment (KEV/known exploited).
> Precedence affects **Affected** ranges and **fixed** info; **severity** is normalized to the **maximum** credible severity unless policy overrides. Conflicts are retained with **source provenance**.
---
## 4) Connectors & normalization
### 4.1 Connector contract
```csharp
public interface IFeedConnector {
string SourceName { get; }
Task FetchAsync(IServiceProvider sp, CancellationToken ct); // -> document collection
Task ParseAsync(IServiceProvider sp, CancellationToken ct); // -> dto collection (validated)
Task MapAsync(IServiceProvider sp, CancellationToken ct); // -> advisory/alias/affected/reference
}
```
* **Fetch**: windowed (cursor), conditional GET (ETag/LastModified), retry/backoff, rate limiting.
* **Parse**: schema validation (JSON Schema, XSD/CSAF), content type checks; write **DTO** with normalized casing.
* **Map**: build canonical records; all outputs carry **provenance** (doc digest, URI, anchors).
### 4.2 Version range normalization
* **SemVer** ecosystems (npm, pypi, maven, nuget, golang): normalize to `introduced`/`fixed` semver ranges (use `~`, `^`, `<`, `>=` canonicalized to intervals).
* **RPM EVR**: `epoch:version-release` with `rpmvercmp` semantics; store raw EVR strings and also **computed order keys** for query.
* **DEB**: dpkg version comparison semantics mirrored; store computed keys.
* **APK**: Alpine version semantics; compute order keys.
* **Generic**: if provider uses text, retain raw; do **not** invent ranges.
### 4.3 Severity & CVSS
* Normalize **CVSS v2/v3/v4** where available (vector, baseScore, severity).
* If multiple CVSS sources exist, track them all; **effective severity** defaults to **max** by policy (configurable).
* **ExploitKnown** toggled by KEV and equivalent sources; store **evidence** (source, date).
---
## 5) Merge engine
### 5.1 Keying & identity
* Identity graph: **CVE** is primary node; vendor/distro IDs resolved via **Alias** edges (from connectors and Conceliers alias tables).
* `advisoryKey` is the canonical primary key (CVE if present, else vendor/distro key).
### 5.2 Merge algorithm (deterministic)
1. **Gather** all rows for `advisoryKey` (across sources).
2. **Select title/summary** by precedence source (vendor>distro>ecosystem>cert).
3. **Union aliases** (dedupe by scheme+value).
4. **Merge `Affected`** with rules:
* Prefer **vendor** ranges for vendor products; prefer **distro** for **distroshipped** packages.
* If both exist for same `productKey`, keep **both**; mark `sourceTag` and `precedence` so **Policy** can decide.
* Never collapse range semantics across different families (e.g., rpm EVR vs semver).
5. **CVSS/severity**: record all CVSS sets; compute **effectiveSeverity** = max (unless policy override).
6. **References**: union with type precedence (advisory > patch > kb > exploit > blog); dedupe by URL; preserve `sourceTag`.
7. Produce **canonical JSON**; compute **afterHash**; store **MergeEvent** with inputs and hashes.
> The merge is **pure** given inputs. Any change in inputs or precedence matrices changes the **hash** predictably.
---
## 6) Storage schema (MongoDB)
**Collections & indexes**
* `source` `{_id, type, baseUrl, enabled, notes}`
* `source_state` `{sourceName(unique), enabled, cursor, lastSuccess, backoffUntil, paceOverrides}`
* `document` `{_id, sourceName, uri, fetchedAt, sha256, contentType, status, metadata, gridFsId?, etag?, lastModified?}`
* Index: `{sourceName:1, uri:1}` unique, `{fetchedAt:-1}`
* `dto` `{_id, sourceName, documentId, schemaVer, payload, validatedAt}`
* Index: `{sourceName:1, documentId:1}`
* `advisory` `{_id, advisoryKey, title, summary, published, modified, severity, cvss, exploitKnown, sources[]}`
* Index: `{advisoryKey:1}` unique, `{modified:-1}`, `{severity:1}`, text index (title, summary)
* `alias` `{advisoryId, scheme, value}`
* Index: `{scheme:1,value:1}`, `{advisoryId:1}`
* `affected` `{advisoryId, productKey, rangeKind, introduced?, fixed?, arch?, distro?, ecosystem?}`
* Index: `{productKey:1}`, `{advisoryId:1}`, `{productKey:1, rangeKind:1}`
* `reference` `{advisoryId, url, kind, sourceTag}`
* Index: `{advisoryId:1}`, `{kind:1}`
* `merge_event` `{advisoryKey, beforeHash, afterHash, mergedAt, inputs[]}`
* Index: `{advisoryKey:1, mergedAt:-1}`
* `export_state` `{_id(exportKind), baseExportId?, baseDigest?, lastFullDigest?, lastDeltaDigest?, cursor, files[]}`
* `locks` `{_id(jobKey), holder, acquiredAt, heartbeatAt, leaseMs, ttlAt}` (TTL cleans dead locks)
* `jobs` `{_id, type, args, state, startedAt, heartbeatAt, endedAt, error}`
**GridFS buckets**: `fs.documents` for raw payloads.
---
## 7) Exporters
### 7.1 Deterministic JSON (vulnlist style)
* Folder structure mirroring `/<scheme>/<first-two>/<rest>/…` with one JSON per advisory; deterministic ordering, stable timestamps, normalized whitespace.
* `manifest.json` lists all files with SHA256 and a toplevel **export digest**.
### 7.2 Trivy DB exporter
* Builds Bolt DB archives compatible with Trivy; supports **full** and **delta** modes.
* In delta, unchanged blobs are reused from the base; metadata captures:
```
{
"mode": "delta|full",
"baseExportId": "...",
"baseManifestDigest": "sha256:...",
"changed": ["path1", "path2"],
"removed": ["path3"]
}
```
* Optional ORAS push (OCI layout) for registries.
* Offline kit bundles include Trivy DB + JSON tree + export manifest.
### 7.3 Handoff to Signer/Attestor (optional)
* On export completion, if `attest: true` is set in job args, Concelier **posts** the artifact metadata to **Signer**/**Attestor**; Concelier itself **does not** hold signing keys.
* Export record stores returned `{ uuid, index, url }` from **Rekor v2**.
---
## 8) REST APIs
All under `/api/v1/concelier`.
**Health & status**
```
GET /healthz | /readyz
GET /status → sources, last runs, export cursors
```
**Sources & jobs**
```
GET /sources → list of configured sources
POST /sources/{name}/trigger → { jobId }
POST /sources/{name}/pause | /resume → toggle
GET /jobs/{id} → job status
```
**Exports**
```
POST /exports/json { full?:bool, force?:bool, attest?:bool } → { exportId, digest, rekor? }
POST /exports/trivy { full?:bool, force?:bool, publish?:bool, attest?:bool } → { exportId, digest, rekor? }
GET /exports/{id} → export metadata (kind, digest, createdAt, rekor?)
```
**Search (operator debugging)**
```
GET /advisories/{key}
GET /advisories?scheme=CVE&value=CVE-2025-12345
GET /affected?productKey=pkg:rpm/openssl&limit=100
```
**AuthN/Z:** Authority tokens (OpTok) with roles: `concelier.read`, `concelier.admin`, `concelier.export`.
---
## 9) Configuration (YAML)
```yaml
concelier:
mongo: { uri: "mongodb://mongo/concelier" }
s3:
endpoint: "http://minio:9000"
bucket: "stellaops-concelier"
scheduler:
windowSeconds: 30
maxParallelSources: 4
sources:
- name: redhat
kind: csaf
baseUrl: https://access.redhat.com/security/data/csaf/v2/
signature: { type: pgp, keys: [ "…redhat PGP…" ] }
enabled: true
windowDays: 7
- name: suse
kind: csaf
baseUrl: https://ftp.suse.com/pub/projects/security/csaf/
signature: { type: pgp, keys: [ "…suse PGP…" ] }
- name: ubuntu
kind: usn-json
baseUrl: https://ubuntu.com/security/notices.json
signature: { type: none }
- name: osv
kind: osv
baseUrl: https://api.osv.dev/v1/
signature: { type: none }
- name: ghsa
kind: ghsa
baseUrl: https://api.github.com/graphql
auth: { tokenRef: "env:GITHUB_TOKEN" }
exporters:
json:
enabled: true
output: s3://stellaops-concelier/json/
trivy:
enabled: true
mode: full
output: s3://stellaops-concelier/trivy/
oras:
enabled: false
repo: ghcr.io/org/concelier
precedence:
vendorWinsOverDistro: true
distroWinsOverOsv: true
severity:
policy: max # or 'vendorPreferred' / 'distroPreferred'
```
---
## 10) Security & compliance
* **Outbound allowlist** per connector (domains, protocols); proxy support; TLS pinning where possible.
* **Signature verification** for raw docs (PGP/cosign/x509) with results stored in `document.metadata.sig`. Docs failing verification may still be ingested but flagged; **merge** can downweight or ignore them by config.
* **No secrets in logs**; auth material via `env:` or mounted files; HTTP redaction of `Authorization` headers.
* **Multitenant**: pertenant DBs or prefixes; pertenant S3 prefixes; tenantscoped API tokens.
* **Determinism**: canonical JSON writer; export digests stable across runs given same inputs.
---
## 11) Performance targets & scale
* **Ingest**: ≥ 5k documents/min on 4 cores (CSAF/OpenVEX/JSON).
* **Normalize/map**: ≥ 50k `Affected` rows/min on 4 cores.
* **Merge**: ≤ 10ms P95 per advisory at steadystate updates.
* **Export**: 1M advisories JSON in ≤ 90s (streamed, zstd), Trivy DB in ≤ 60s on 8 cores.
* **Memory**: hard cap per job; chunked streaming writers; backpressure to avoid GC spikes.
**Scale pattern**: add Concelier replicas; Mongo scaling via indices and read/write concerns; GridFS only for oversized docs.
---
## 12) Observability
* **Metrics**
* `concelier.fetch.docs_total{source}`
* `concelier.fetch.bytes_total{source}`
* `concelier.parse.failures_total{source}`
* `concelier.map.affected_total{source}`
* `concelier.merge.changed_total`
* `concelier.export.bytes{kind}`
* `concelier.export.duration_seconds{kind}`
* **Tracing** around fetch/parse/map/merge/export.
* **Logs**: structured with `source`, `uri`, `docDigest`, `advisoryKey`, `exportId`.
---
## 13) Testing matrix
* **Connectors:** fixture suites for each provider/format (happy path; malformed; signature fail).
* **Version semantics:** EVR vs dpkg vs semver edge cases (epoch bumps, tilde versions, prereleases).
* **Merge:** conflicting sources (vendor vs distro vs OSV); verify precedence & dual retention.
* **Export determinism:** byteforbyte stable outputs across runs; digest equality.
* **Performance:** soak tests with 1M advisories; cap memory; verify backpressure.
* **API:** pagination, filters, RBAC, error envelopes (RFC 7807).
* **Offline kit:** bundle build & import correctness.
---
## 14) Failure modes & recovery
* **Source outages:** scheduler backs off with exponential delay; `source_state.backoffUntil`; alerts on staleness.
* **Schema drifts:** parse stage marks DTO invalid; job fails with clear diagnostics; connector version flags track supported schema ranges.
* **Partial exports:** exporters write to temp prefix; **manifest commit** is atomic; only then move to final prefix and update `export_state`.
* **Resume:** all stages idempotent; `source_state.cursor` supports window resume.
---
## 15) Operator runbook (quick)
* **Trigger all sources:** `POST /api/v1/concelier/sources/*/trigger`
* **Force full export JSON:** `POST /api/v1/concelier/exports/json { "full": true, "force": true }`
* **Force Trivy DB delta publish:** `POST /api/v1/concelier/exports/trivy { "full": false, "publish": true }`
* **Inspect advisory:** `GET /api/v1/concelier/advisories?scheme=CVE&value=CVE-2025-12345`
* **Pause noisy source:** `POST /api/v1/concelier/sources/osv/pause`
---
## 16) Rollout plan
1. **MVP**: Red Hat (CSAF), SUSE (CSAF), Ubuntu (USN JSON), OSV; JSON export.
2. **Add**: GHSA GraphQL, Debian (DSA HTML/JSON), Alpine secdb; Trivy DB export.
3. **Attestation handoff**: integrate with **Signer/Attestor** (optional).
4. **Scale & diagnostics**: provider dashboards, staleness alerts, export cache reuse.
5. **Offline kit**: endtoend verified bundles for airgap.

462
docs/ARCHITECTURE_DEVOPS.md Normal file
View File

@@ -0,0 +1,462 @@
# component_architecture_devops.md — **StellaOps Release & Operations** (2025Q4)
> **Scope.** Implementationready blueprint for **how StellaOps is built, versioned, signed, distributed, upgraded, licensed (PoE)**, and operated in customer environments (online and airgapped). Covers reproducible builds, supplychain attestations, registries, offline kits, migration/rollback, artifact lifecycle (MinIO/Mongo), monitoring SLOs, and customer activation.
---
## 0) Product vision (operations lens)
StellaOps must be **trustable at a glance** and **boringly operable**:
* Every release ships with **firstparty SBOMs, provenance, and signatures**; services verify **each others** integrity at runtime.
* Customers can deploy by **digest** and stay aligned with **LTS/stable/edge** channels.
* Paid customers receive **attestation authority** (Signer accepts their PoE) while the core platform remains **free to run**.
* Airgapped customers receive **offline kits** with verifiable digests and deterministic import.
* Artifacts expire predictably; operators know whats kept, for how long, and why.
---
## 1) Release trains & versioning
### 1.1 Channels
* **LTS** (12month support window): quarterly cadence (Q1/Q2/Q3/Q4).
* **Stable** (default): monthly rollup (bug fixes + compatible features).
* **Edge**: weekly; for early adopters, no guarantees.
### 1.2 Version strings
Semantic core + calendar tag:
```
<MAJOR>.<MINOR>.<PATCH> (<YYYY>.<MM>) e.g., 2.4.1 (2027.06)
```
* **MAJOR**: breaking API/DB changes (rare).
* **MINOR**: new features, compatible schema migrations (expand/contract pattern).
* **PATCH**: bug fixes, perf and security updates.
* **Calendar tag** exposes **release year** used by Signer for **PoE window checks**.
### 1.3 Component alignment
A release is a **bundle** of image digests + charts + manifests. All services in a bundle are **wirecompatible**. Mixed minor versions are allowed within a bounded skew:
* **Web UI ↔ backend**: `±1 minor`.
* **Scanner ↔ Policy/Excititor/Concelier**: `±1 minor`.
* **Authority/Signer/Attestor triangle**: **must** be same minor (crypto and DPoP/mTLS binding rules).
At startup, services **selfadvertise** their semver & channel; the UI surfaces **mismatch warnings**.
---
## 2) Supplychain pipeline (how a release is built)
### 2.1 Deterministic builds
* **Builders**: isolated **BuildKit** workers with pinned base images (digest only).
* **Pinning**: lock files or `go.mod`, `package-lock.json`, `global.json`, `Directory.Packages.props` are **frozen** at tag.
* **Reproducibility**: timestamps normalized; source date epoch; deterministic zips/tars.
* **Multiarch**: linux/amd64 + linux/arm64 (Windows images track M2 roadmap).
### 2.2 Firstparty SBOMs & provenance
* Each image gets **CycloneDX (JSON+Protobuf) SBOM** and **SLSAstyle provenance** attached as **OCI referrers**.
* Scanners **Buildx generator** is used to produce SBOMs *during* build; a separate postbuild scan verifies parity (red flag if drift).
* **Release manifest** (see §6.1) lists all digests and SBOM/attestation refs.
### 2.3 Signing & transparency
* Images are **cosignsigned** (keyless) with a StellaOps release identity; inclusion in a **transparency log** (Rekor) is required.
* SBOM and provenance attestations are **DSSE** and also transparencylogged.
* Release keys (Fulcio roots or public keys) are embedded in **Signer** policy (for **scannerrelease validation** at customer side).
### 2.4 Gates & tests
* **Static**: linters, codegen checks, protobuf API freeze (backwardcompat tests).
* **Unit/integration**: percomponent, plus **endtoend** flows (scan→vex→policy→sign→attest).
* **Perf SLOs**: hot paths (SBOM compose, diff, export) measured against budgets.
* **Security**: dependency audit vs Concelier export; container hardening tests; minimal caps.
* **Canary cohort**: internal staging + selected customers; one week on **edge** before **stable** tag.
---
## 3) Distribution & activation
### 3.1 Registries
* **Primary**: `registry.stella-ops.org` (OCI v2, supports Referrers API).
* **Mirrors**: GHCR (readonly), regional mirrors for latency.
* **Pull by digest only** in Kubernetes/Compose manifests.
**Gating policy**:
* **Core images** (Authority, Scanner, Concelier, Excititor, Attestor, UI): public **read**.
* **Enterprise addons** (if any) and **prerelease**: private repos via OAuth2 token service.
> Monetization lever is **signing** (PoE gate), not image pulls, so the core remains simple to consume.
### 3.2 OAuth2 token service (for private repos)
* Docker Registrys token flow backed by **Authority**:
1. Client hits registry (`401` with `WWW-Authenticate: Bearer realm=…`).
2. Client gets an **access token** from the token service (validated by Authority) with `scope=repository:…:pull`.
3. Registry allows pull for the requested repo.
* Tokens are **shortlived** (60300s) and **DPoPbound**.
### 3.3 Offline kits (airgapped)
* Tarball per release channel:
```
stellaops-kit-<ver>-<channel>.tar.zst
/images/ OCI layout with all first-party images (multi-arch)
/sboms/ CycloneDX JSON+PB for each image
/attest/ DSSE bundles + Rekor proofs
/charts/ Helm charts + values templates
/compose/ docker-compose.yml + .env template
/plugins/ Concelier/Excititor connectors (restart-time)
/policy/ example policies
/manifest/ release.yaml (see §6.1)
```
* Import via CLI `offline kit import`; checks digests and signatures before load.
---
## 4) Licensing (PoE) & monetization
**Principle**: **Only paid StellaOps issues valid signed attestations.** Running the stack is free; signing requires PoE.
### 4.1 PoE issuance
* Customers purchase a plan and obtain a **PoE artifact** from `www.stella-ops.org`:
* **PoEJWT** (DPoP/mTLSbound) **or** **PoE mTLS client certificate**.
* Contains: `license_id`, `plan`, `valid_release_year`, `max_version`, `exp`, optional `tenant/customer` IDs.
### 4.2 Online enforcement
* **Signer** calls **Licensing /license/introspect** on every signing request (see signer doc).
* If **revoked/expired/outofwindow** → deny with machinereadable reason.
* All **valid** bundles are DSSEsigned and **Attestor** logs them; Rekor UUID returned.
* UI badges: “**Verified by StellaOps**” with link to the public log.
### 4.3 Airgapped / offline
* Customers obtain a **timeboxed PoE lease** (signed JSON, 730 days).
* Signer accepts the lease and emits **provisional** attestations (clearly labeled).
* When connectivity returns, a background job **endorses** the provisional entries with the cloud service, updating their status to **verified**.
* Operators can export a **verification bundle** for auditors even before endorsement (contains DSSE + local Rekor proof + lease snapshot).
### 4.4 Stolen/abused PoE
* Customers report theft; **Licensing** flags `license_id` as **revoked**.
* Subsequent Signer requests **deny**; previous attestations remain but can be marked **contested** (UI shows badge, optional resign path upon new PoE).
---
## 5) Deployment path (customer side)
### 5.1 First install
* **Helm** (Kubernetes) or **Compose** (VMs). Example (K8s):
```bash
helm repo add stellaops https://charts.stella-ops.org
helm install stella stellaops/platform \
--version 2.4.0 \
--set global.channel=stable \
--set authority.issuer=https://authority.stella.local \
--set scanner.minio.endpoint=http://minio.stella.local:9000 \
--set scanner.mongo.uri=mongodb://mongo/scanner \
--set concelier.mongo.uri=mongodb://mongo/concelier \
--set excititor.mongo.uri=mongodb://mongo/excititor
```
* Postinstall job registers **Authority clients** (Scanner, Signer, Attestor, UI) and prints **bootstrap** URLs and client credentials (sealed secrets).
* UI banner shows **release bundle** and verification state (cosign OK? Rekor OK?).
### 5.2 Updates
* **Blue/green**: pull new bundle by **digest**; deploy sidebyside; cut traffic.
* **Rolling**: upgrade stateful components in safe order:
1. Authority (stateless, dualkey rotation ready)
2. Signer/Attestor (same minor)
3. Scanner WebService & Workers
4. Concelier, then Excititor (schema migrations are expand/contract)
5. UI last
* **DB migrations** are **expand/contract**:
* Phase A (release N): **add** new fields/indexes, write old+new.
* Phase B (N+1): **read** new fields; **drop** old.
* Rollback is a matter of redeploying previous images and keeping both schemas valid.
### 5.3 Rollback
* Images referenced by **digest**; keep previous release manifest `K` versions back.
* `helm rollback` or compose `docker compose -f release-K.yml up -d`.
* Mongo migrations are additive; **no destructive changes** within a single minor.
---
## 6) Release payloads & manifests
### 6.1 Release manifest (`release.yaml`)
```yaml
release:
version: "2.4.1"
channel: "stable"
date: "2027-06-20T12:00:00Z"
calendar: "2027.06"
components:
- name: scanner-webservice
image: registry.stella-ops.org/stellaops/scanner-web@sha256:aa..bb
sbom: oci://.../referrers/cdx-json@sha256:11..22
provenance: oci://.../attest/provenance@sha256:33..44
signature: { rekorUUID: "…" }
- name: signer
image: registry.stella-ops.org/stellaops/signer@sha256:cc..dd
signature: { rekorUUID: "…" }
charts:
- name: platform
version: "2.4.1"
digest: "sha256:ee..ff"
compose:
file: "docker-compose.yml"
digest: "sha256:77..88"
checksums:
sha256: "… digest of this release.yaml …"
```
The manifest is **cosignsigned**; UI/CLI can verify a bundle without talking to registries.
### 6.2 Image labels (release metadata)
Each image sets OCI labels:
```
org.opencontainers.image.version = "2.4.1"
org.opencontainers.image.revision = "<git sha>"
org.opencontainers.image.created = "2027-06-20T12:00:00Z"
org.stellaops.release.calendar = "2027.06"
org.stellaops.release.channel = "stable"
org.stellaops.build.slsaProvenance = "oci://…"
```
Signer validates **scanner** images cosign identity + calendar tag for **release window** checks.
---
## 7) Artifact lifecycle & storage (MinIO/Mongo)
### 7.1 Buckets & prefixes (MinIO)
```
s3://stellaops/
scanner/
layers/<sha256>/sbom.cdx.json.zst
images/<imgDigest>/inventory.cdx.pb
images/<imgDigest>/usage.cdx.pb
diffs/<old>_<new>/diff.json.zst
attest/<artifactSha256>.dsse.json
concelier/
json/<exportId>/...
trivy/<exportId>/...
excititor/
exports/<exportId>/...
attestor/
dsse/<bundleSha256>.json
proof/<rekorUuid>.json
```
### 7.2 ILM classes
* **`short`**: working artifacts (diffs, queues) — TTL 714 days.
* **`default`**: SBOMs & indexes — TTL 90180 days (configurable).
* **`compliance`**: signed reports & attested exports — **Object Lock** (governance/compliance) 17 years.
### 7.3 Artifact Lifecycle Controller (ALC)
* A background worker (part of Scanner.WebService) enforces **TTL** and **reference counting**:
* Artifacts referenced by **reports** or **tickets** are pinned.
* ILM actions logged; UI shows perclass usage & upcoming purges.
### 7.4 Mongo retention
* **Scanner**: `runtime.events` use TTL (e.g., 3090 days); **catalog** permanent.
* **Concelier/Excititor**: raw docs keep **last N windows**; canonical stores permanent.
* **Attestor**: `entries` permanent; `dedupe` TTL 2448h.
---
## 8) Observability & SLOs (operations)
* **Uptime SLO**: 99.9% for Signer/Authority/Attestor; 99.5% for Scanner WebService; Excititor/Concelier 99.0%.
* **Error budgets**: tracked per month; dashboards show burn rates.
* **Golden signals**:
* **Latency**: token issuance, sign→attest roundtrip, scan enqueue→emit, export build.
* **Saturation**: queue depth, Mongo write IOPS, MinIO net throughput.
* **Traffic**: scans/min, attestations/min, webhook admits/min.
* **Errors**: 5xx rates, cosign verification failures, Rekor timeouts.
Prometheus + OTLP; Grafana dashboards ship in the charts.
---
## 9) Security & compliance operations
* **Key rotation**:
* Authority JWKS: 60day cadence, dualkey overlap.
* Release signing identities: rotate per minor or quarterly.
* Sigstore roots mirrored and pinned; alarms on drift.
* **FIPS mode** (Gov build):
* Enforce `ES256` + KMS/HSM; disable Ed25519; MLS ciphers only.
* Local **Rekor v2** and **Fulcio** alternatives; **airgapped** CA.
* **Vulnerability response**:
* Concelier redflag advisories trigger accelerated **stable** patch rollout; UI/CLI “security patch available” notice.
* **Backups/DR**:
* Mongo nightly snapshots; MinIO versioning + replication (if configured).
* Restore runbooks tested quarterly with synthetic data.
---
## 10) Customer update flow (how versions are fetched & activated)
### 10.1 Online clusters
* **UI** surfaces update banner with **release manifest** diff and risk notes.
* Operator approves → **Controller** pulls new images by digest; healthchecks; moves traffic; deprecates old revision.
* Postswitch, **schema Phase B** migrations (if any) run automatically.
### 10.2 Airgapped clusters
* Operator downloads **offline kit** from a mirror → `stellaops offline kit import`.
* Controller validates bundle checksums and **cosign signatures**; applies charts/compose by digest.
* After install, **verify** page shows green checks: image sigs, SBOMs attached, provenance logged.
### 10.3 CLI selfupdate (optional)
* `stellaops self-update` pulls a **signed release manifest** and verifies the **CLI binary** with cosign before swapping (admin can disable).
---
## 11) Compatibility & deprecation policy
* **APIs** are stable within a **major**; breaking changes imply **MAJOR++** and deprecation period of one minor.
* **Storage**: expand/contract; “drop old fields” only after one minor grace.
* **Config**: feature flags (default off) for risky features (e.g., eBPF).
---
## 12) Runbooks (selected)
### 12.1 Lost PoE
1. Suspend **automatic attestation** jobs.
2. Use CLI `stellaops signer status` to confirm `entitlement_denied`.
3. Obtain new PoE from portal; verify on Signer `/poe/verify`.
4. Reenable; optionally **resign** last N reports (UI button → batch).
### 12.2 Rekor outage (selfhosted)
* Attestor returns `202 (pending)` with queued proof fetch.
* Keep DSSE bundles locally; resubmit on schedule; UI badge shows **Pending**.
* If outage > SLA, you can switch to a **mirror** log in config; Attestor writes to both when restored.
### 12.3 Emergency downgrade
* Identify prior release manifest (UI → Admin → Releases).
* `helm rollback stella <revision>` (or compose apply previous file).
* Services tolerate skew per §1.3; ensure **Signer/Authority/Attestor** are rolled together.
---
## 13) Example: cluster bootstrap (Compose)
```yaml
version: "3.9"
services:
authority:
image: registry.stella-ops.org/stellaops/authority@sha256:...
env_file: ./env/authority.env
ports: ["8440:8440"]
signer:
image: registry.stella-ops.org/stellaops/signer@sha256:...
depends_on: [authority]
environment:
- SIGNER__POE__LICENSING__INTROSPECTURL=https://www.stella-ops.org/api/v1/license/introspect
attestor:
image: registry.stella-ops.org/stellaops/attestor@sha256:...
depends_on: [signer]
scanner-web:
image: registry.stella-ops.org/stellaops/scanner-web@sha256:...
environment:
- SCANNER__S3__ENDPOINT=http://minio:9000
scanner-worker:
image: registry.stella-ops.org/stellaops/scanner-worker@sha256:...
deploy: { replicas: 4 }
concelier:
image: registry.stella-ops.org/stellaops/concelier@sha256:...
excititor:
image: registry.stella-ops.org/stellaops/excititor@sha256:...
web-ui:
image: registry.stella-ops.org/stellaops/web-ui@sha256:...
mongo:
image: mongo:7
minio:
image: minio/minio:RELEASE.2025-07-10T00-00-00Z
```
---
## 14) Governance & keys (who owns the trust root)
* **Release key policy**: only the Release Engineering group can push signed releases; 4eyes approval; TUFstyle manifest possible in future.
* **Signer acceptance policy**: embedded release identities are updated **only** via minor upgrade; emergency CRL supported.
* **Customer keys**: none needed for core use; enterprise addons may require percustomer registries and keys.
---
## 15) Roadmap (Ops)
* **Windows containers GA** (Scanner + Zastava).
* **Key Transparency** for Signer certs.
* **Deltakit** (offline) for incremental updates.
* **Operator CRDs** (K8s) to manage policy and ILM declaratively.
* **SBOM **protobuf** as default transport at rest (smaller, faster).
---
### Appendix A — Minimal SLO monitors
* `authority.tokens_issued_total` slope ≈ normal.
* `signer.requests_total{result="success"}/minute` > 0 (when scans occur).
* `attestor.submit_latency_seconds{quantile=0.95}` < 0.3.
* `scanner.scan_latency_seconds{quantile=0.95}` < target per image size.
* `concelier.export.duration_seconds` stable; `excititor.consensus.conflicts_total` not exploding after policy changes.
* MinIO `s3_requests_errors_total` near zero; Mongo `opcounters` hit expected baseline.
### Appendix B — Upgrade safety checklist
* Verify **release manifest** signature.
* Ensure **Signer/Authority/Attestor** are same minor.
* Verify **DB backups** < 24h old.
* Confirm **ILM** wont purge compliance artifacts during upgrade window.
* Roll **one component** at a time; watch SLOs; abort on regression.
---
**End — component_architecture_devops.md**

View File

@@ -0,0 +1,463 @@
# component_architecture_excititor.md — **StellaOps Excititor** (2025Q4)
> **Scope.** This document specifies the **Excititor** service: its purpose, trust model, data structures, APIs, plugin contracts, storage schema, normalization/consensus algorithms, performance budgets, testing matrix, and how it integrates with Scanner, Policy, Concelier, and the attestation chain. It is implementationready.
---
## 0) Mission & role in the platform
**Mission.** Convert heterogeneous **VEX** statements (OpenVEX, CSAF VEX, CycloneDX VEX; vendor/distro/platform sources) into **canonical, queryable claims**; compute **deterministic consensus** per *(vuln, product)*; preserve **conflicts with provenance**; publish **stable, attestable exports** that the backend uses to suppress nonexploitable findings, prioritize remaining risk, and explain decisions.
**Boundaries.**
* Excititor **does not** decide PASS/FAIL. It supplies **evidence** (statuses + justifications + provenance weights).
* Excititor preserves **conflicting claims** unchanged; consensus encodes how we would pick, but the raw set is always exportable.
* VEX consumption is **backendonly**: Scanner never applies VEX. The backends **Policy Engine** asks Excititor for status evidence and then decides what to show.
---
## 1) Inputs, outputs & canonical domain
### 1.1 Accepted input formats (ingest)
* **OpenVEX** JSON documents (attested or raw).
* **CSAF VEX** 2.x (vendor PSIRTs and distros commonly publish CSAF).
* **CycloneDX VEX** 1.4+ (standalone VEX or embedded VEX blocks).
* **OCIattached attestations** (VEX statements shipped as OCI referrers) — optional connectors.
All connectors register **source metadata**: provider identity, trust tier, signature expectations (PGP/cosign/PKI), fetch windows, rate limits, and time anchors.
### 1.2 Canonical model (normalized)
Every incoming statement becomes a set of **VexClaim** records:
```
VexClaim
- providerId // 'redhat', 'suse', 'ubuntu', 'github', 'vendorX'
- vulnId // 'CVE-2025-12345', 'GHSA-xxxx', canonicalized
- productKey // canonical product identity (see §2.2)
- status // affected | not_affected | fixed | under_investigation
- justification? // for 'not_affected'/'affected' where provided
- introducedVersion? // semantics per provider (range or exact)
- fixedVersion? // where provided (range or exact)
- lastObserved // timestamp from source or fetch time
- provenance // doc digest, signature status, fetch URI, line/offset anchors
- evidence[] // raw source snippets for explainability
- supersedes? // optional cross-doc chain (docDigest → docDigest)
```
### 1.3 Exports (consumption)
* **VexConsensus** per `(vulnId, productKey)` with:
* `rollupStatus` (after policy weights/justification gates),
* `sources[]` (winning + losing claims with weights & reasons),
* `policyRevisionId` (identifier of the Excititor policy used),
* `consensusDigest` (stable SHA256 over canonical JSON).
* **Raw claims** export for auditing (unchanged, with provenance).
* **Provider snapshots** (per source, last N days) for operator debugging.
* **Index** optimized for backend joins: `(productKey, vulnId) → (status, confidence, sourceSet)`.
All exports are **deterministic**, and (optionally) **attested** via DSSE and logged to Rekor v2.
---
## 2) Identity model — products & joins
### 2.1 Vuln identity
* Accepts **CVE**, **GHSA**, vendor IDs (MSRC, RHSA…), distro IDs (DSA/USN/RHSA…) — normalized to `vulnId` with alias sets.
* **Alias graph** maintained (from Concelier) to map vendor/distro IDs → CVE (primary) and to **GHSA** where applicable.
### 2.2 Product identity (`productKey`)
* **Primary:** `purl` (Package URL).
* **Secondary links:** `cpe`, **OS package NVRA/EVR**, NuGet/Maven/Golang identity, and **OS package name** when purl unavailable.
* **Fallback:** `oci:<registry>/<repo>@<digest>` for imagelevel VEX.
* **Special cases:** kernel modules, firmware, platforms → providerspecific mapping helpers (connector captures providers product taxonomy → canonical `productKey`).
> Excititor does not invent identities. If a provider cannot be mapped to purl/CPE/NVRA deterministically, we keep the native **product string** and mark the claim as **nonjoinable**; the backend will ignore it unless a policy explicitly whitelists that provider mapping.
---
## 3) Storage schema (MongoDB)
Database: `excititor`
### 3.1 Collections
**`vex.providers`**
```
_id: providerId
name, homepage, contact
trustTier: enum {vendor, distro, platform, hub, attestation}
signaturePolicy: { type: pgp|cosign|x509|none, keys[], certs[], cosignKeylessRoots[] }
fetch: { baseUrl, kind: http|oci|file, rateLimit, etagSupport, windowDays }
enabled: bool
createdAt, modifiedAt
```
**`vex.raw`** (immutable raw documents)
```
_id: sha256(doc bytes)
providerId
uri
ingestedAt
contentType
sig: { verified: bool, method: pgp|cosign|x509|none, keyId|certSubject, bundle? }
payload: GridFS pointer (if large)
disposition: kept|replaced|superseded
correlation: { replaces?: sha256, replacedBy?: sha256 }
```
**`vex.claims`** (normalized rows; dedupe on providerId+vulnId+productKey+docDigest)
```
_id
providerId
vulnId
productKey
status
justification?
introducedVersion?
fixedVersion?
lastObserved
docDigest
provenance { uri, line?, pointer?, signatureState }
evidence[] { key, value, locator }
indices:
- {vulnId:1, productKey:1}
- {providerId:1, lastObserved:-1}
- {status:1}
- text index (optional) on evidence.value for debugging
```
**`vex.consensus`** (rollups)
```
_id: sha256(canonical(vulnId, productKey, policyRevision))
vulnId
productKey
rollupStatus
sources[]: [
{ providerId, status, justification?, weight, lastObserved, accepted:bool, reason }
]
policyRevisionId
evaluatedAt
consensusDigest // same as _id
indices:
- {vulnId:1, productKey:1}
- {policyRevisionId:1, evaluatedAt:-1}
```
**`vex.exports`** (manifest of emitted artifacts)
```
_id
querySignature
format: raw|consensus|index
artifactSha256
rekor { uuid, index, url }?
createdAt
policyRevisionId
cacheable: bool
```
**`vex.cache`**
```
querySignature -> exportId (for fast reuse)
ttl, hits
```
**`vex.migrations`**
* ordered migrations applied at bootstrap to ensure indexes.
### 3.2 Indexing strategy
* Hot path queries use exact `(vulnId, productKey)` and timebounded windows; compound indexes cover both.
* Providers list view by `lastObserved` for monitoring staleness.
* `vex.consensus` keyed by `(vulnId, productKey, policyRevision)` for deterministic reuse.
---
## 4) Ingestion pipeline
### 4.1 Connector contract
```csharp
public interface IVexConnector
{
string ProviderId { get; }
Task FetchAsync(VexConnectorContext ctx, CancellationToken ct); // raw docs
Task NormalizeAsync(VexConnectorContext ctx, CancellationToken ct); // raw -> VexClaim[]
}
```
* **Fetch** must implement: window scheduling, conditional GET (ETag/IfModifiedSince), rate limiting, retry/backoff.
* **Normalize** parses the format, validates schema, maps product identities deterministically, emits `VexClaim` records with **provenance**.
### 4.2 Signature verification (per provider)
* **cosign (keyless or keyful)** for OCI referrers or HTTPserved JSON with Sigstore bundles.
* **PGP** (provider keyrings) for distro/vendor feeds that sign docs.
* **x509** (mutual TLS / providerpinned certs) where applicable.
* Signature state is stored on **vex.raw.sig** and copied into **provenance.signatureState** on claims.
> Claims from sources failing signature policy are marked `"signatureState.verified=false"` and **policy** can downweight or ignore them.
### 4.3 Time discipline
* For each doc, prefer **providers document timestamp**; if absent, use fetch time.
* Claims carry `lastObserved` which drives **tiebreaking** within equal weight tiers.
---
## 5) Normalization: product & status semantics
### 5.1 Product mapping
* **purl** first; **cpe** second; OS package NVRA/EVR mapping helpers (distro connectors) produce purls via canonical tables (e.g., rpm→purl:rpm, deb→purl:deb).
* Where a provider publishes **platformlevel** VEX (e.g., “RHEL 9 not affected”), connectors expand to known product inventory rules (e.g., map to sets of packages/components shipped in the platform). Expansion tables are versioned and kept per provider; every expansion emits **evidence** indicating the rule applied.
* If expansion would be speculative, the claim remains **platformscoped** with `productKey="platform:redhat:rhel:9"` and is flagged **nonjoinable**; backend can decide to use platform VEX only when Scanner proves the platform runtime.
### 5.2 Status + justification mapping
* Canonical **status**: `affected | not_affected | fixed | under_investigation`.
* **Justifications** normalized to a controlled vocabulary (CISAaligned), e.g.:
* `component_not_present`
* `vulnerable_code_not_in_execute_path`
* `vulnerable_configuration_unused`
* `inline_mitigation_applied`
* `fix_available` (with `fixedVersion`)
* `under_investigation`
* Providers with freetext justifications are mapped by deterministic tables; raw text preserved as `evidence`.
---
## 6) Consensus algorithm
**Goal:** produce a **stable**, explainable `rollupStatus` per `(vulnId, productKey)` given possibly conflicting claims.
### 6.1 Inputs
* Set **S** of `VexClaim` for the key.
* **Excititor policy snapshot**:
* **weights** per provider tier and per provider overrides.
* **justification gates** (e.g., require justification for `not_affected` to be acceptable).
* **minEvidence** rules (e.g., `not_affected` must come from ≥1 vendor or 2 distros).
* **signature requirements** (e.g., require verified signature for fixed to be considered).
### 6.2 Steps
1. **Filter invalid** claims by signature policy & justification gates → set `S'`.
2. **Score** each claim:
`score = weight(provider) * freshnessFactor(lastObserved)` where freshnessFactor ∈ [0.8, 1.0] for staleness decay (configurable; small effect).
3. **Aggregate** scores per status: `W(status) = Σ score(claims with that status)`.
4. **Pick** `rollupStatus = argmax_status W(status)`.
5. **Tiebreakers** (in order):
* Higher **max single** provider score wins (vendor > distro > platform > hub).
* More **recent** lastObserved wins.
* Deterministic lexicographic order of status (`fixed` > `not_affected` > `under_investigation` > `affected`) as final tiebreaker.
6. **Explain**: mark accepted sources (`accepted=true; reason="weight"`/`"freshness"`), mark rejected sources with explicit `reason` (`"insufficient_justification"`, `"signature_unverified"`, `"lower_weight"`).
> The algorithm is **pure** given S and policy snapshot; result is reproducible and hashed into `consensusDigest`.
---
## 7) Query & export APIs
All endpoints are versioned under `/api/v1/vex`.
### 7.1 Query (online)
```
POST /claims/search
body: { vulnIds?: string[], productKeys?: string[], providers?: string[], since?: timestamp, limit?: int, pageToken?: string }
→ { claims[], nextPageToken? }
POST /consensus/search
body: { vulnIds?: string[], productKeys?: string[], policyRevisionId?: string, since?: timestamp, limit?: int, pageToken?: string }
→ { entries[], nextPageToken? }
POST /resolve
body: { purls: string[], vulnIds: string[], policyRevisionId?: string }
→ { results: [ { vulnId, productKey, rollupStatus, sources[] } ] }
```
### 7.2 Exports (cacheable snapshots)
```
POST /exports
body: { signature: { vulnFilter?, productFilter?, providers?, since? }, format: raw|consensus|index, policyRevisionId?: string, force?: bool }
→ { exportId, artifactSha256, rekor? }
GET /exports/{exportId} → bytes (application/json or binary index)
GET /exports/{exportId}/meta → { signature, policyRevisionId, createdAt, artifactSha256, rekor? }
```
### 7.3 Provider operations
```
GET /providers → provider list & signature policy
POST /providers/{id}/refresh → trigger fetch/normalize window
GET /providers/{id}/status → last fetch, doc counts, signature stats
```
**Auth:** servicetoservice via Authority tokens; operator operations via UI/CLI with RBAC.
---
## 8) Attestation integration
* Exports can be **DSSEsigned** via **Signer** and logged to **Rekor v2** via **Attestor** (optional but recommended for regulated pipelines).
* `vex.exports.rekor` stores `{uuid, index, url}` when present.
* **Predicate type**: `https://stella-ops.org/attestations/vex-export/1` with fields:
* `querySignature`, `policyRevisionId`, `artifactSha256`, `createdAt`.
---
## 9) Configuration (YAML)
```yaml
excititor:
mongo: { uri: "mongodb://mongo/excititor" }
s3:
endpoint: http://minio:9000
bucket: stellaops
policy:
weights:
vendor: 1.0
distro: 0.9
platform: 0.7
hub: 0.5
attestation: 0.6
providerOverrides:
redhat: 1.0
suse: 0.95
requireJustificationForNotAffected: true
signatureRequiredForFixed: true
minEvidence:
not_affected:
vendorOrTwoDistros: true
connectors:
- providerId: redhat
kind: csaf
baseUrl: https://access.redhat.com/security/data/csaf/v2/
signaturePolicy: { type: pgp, keys: [ "…redhat-pgp-key…" ] }
windowDays: 7
- providerId: suse
kind: csaf
baseUrl: https://ftp.suse.com/pub/projects/security/csaf/
signaturePolicy: { type: pgp, keys: [ "…suse-pgp-key…" ] }
- providerId: ubuntu
kind: openvex
baseUrl: https://…/vex/
signaturePolicy: { type: none }
- providerId: vendorX
kind: cyclonedx-vex
ociRef: ghcr.io/vendorx/vex@sha256:…
signaturePolicy: { type: cosign, cosignKeylessRoots: [ "sigstore-root" ] }
```
---
## 10) Security model
* **Input signature verification** enforced per provider policy (PGP, cosign, x509).
* **Connector allowlists**: outbound fetch constrained to configured domains.
* **Tenant isolation**: pertenant DB prefixes or separate DBs; pertenant S3 prefixes; pertenant policies.
* **AuthN/Z**: Authorityissued OpToks; RBAC roles (`vex.read`, `vex.admin`, `vex.export`).
* **No secrets in logs**; deterministic logging contexts include providerId, docDigest, claim keys.
---
## 11) Performance & scale
* **Targets:**
* Normalize 10k VEX claims/minute/core.
* Consensus compute ≤50ms for 1k unique `(vuln, product)` pairs in hot cache.
* Export (consensus) 1M rows in ≤60s on 8 cores with streaming writer.
* **Scaling:**
* WebService handles control APIs; **Worker** background services (same image) execute fetch/normalize in parallel with ratelimits; Mongo writes batched; upserts by natural keys.
* Exports stream straight to S3 (MinIO) with rolling buffers.
* **Caching:**
* `vex.cache` maps query signatures → export; TTL to avoid stampedes; optimistic reuse unless `force`.
---
## 12) Observability
* **Metrics:**
* `vex.ingest.docs_total{provider}`
* `vex.normalize.claims_total{provider}`
* `vex.signature.failures_total{provider,method}`
* `vex.consensus.conflicts_total{vulnId}`
* `vex.exports.bytes{format}` / `vex.exports.latency_seconds`
* **Tracing:** spans for fetch, verify, parse, map, consensus, export.
* **Dashboards:** provider staleness, top conflicting vulns/components, signature posture, export cache hitrate.
---
## 13) Testing matrix
* **Connectors:** golden raw docs → deterministic claims (fixtures per provider/format).
* **Signature policies:** valid/invalid PGP/cosign/x509 samples; ensure rejects are recorded but not accepted.
* **Normalization edge cases:** platformonly claims, freetext justifications, nonpurl products.
* **Consensus:** conflict scenarios across tiers; check tiebreakers; justification gates.
* **Performance:** 1Mrow export timing; memory ceilings; stream correctness.
* **Determinism:** same inputs + policy → identical `consensusDigest` and export bytes.
* **API contract tests:** pagination, filters, RBAC, rate limits.
---
## 14) Integration points
* **Backend Policy Engine** (in Scanner.WebService): calls `POST /resolve` with batched `(purl, vulnId)` pairs to fetch `rollupStatus + sources`.
* **Concelier**: provides alias graph (CVE↔vendor IDs) and may supply VEXadjacent metadata (e.g., KEV flag) for policy escalation.
* **UI**: VEX explorer screens use `/claims/search` and `/consensus/search`; show conflicts & provenance.
* **CLI**: `stellaops vex export --consensus --since 7d --out vex.json` for audits.
---
## 15) Failure modes & fallback
* **Provider unreachable:** stale thresholds trigger warnings; policy can downweight stale providers automatically (freshness factor).
* **Signature outage:** continue to ingest but mark `signatureState.verified=false`; consensus will likely exclude or downweight per policy.
* **Schema drift:** unknown fields are preserved as `evidence`; normalization rejects only on **invalid identity** or **status**.
---
## 16) Rollout plan (incremental)
1. **MVP**: OpenVEX + CSAF connectors for 3 major providers (e.g., Red Hat/SUSE/Ubuntu), normalization + consensus + `/resolve`.
2. **Signature policies**: PGP for distros; cosign for OCI.
3. **Exports + optional attestation**.
4. **CycloneDX VEX** connectors; platform claim expansion tables; UI explorer.
5. **Scale hardening**: export indexes; conflict analytics.
---
## 17) Appendix — canonical JSON (stable ordering)
All exports and consensus entries are serialized via `VexCanonicalJsonSerializer`:
* UTF8 without BOM;
* keys sorted (ASCII);
* arrays sorted by `(providerId, vulnId, productKey, lastObserved)` unless semantic order mandated;
* timestamps in `YYYYMMDDThh:mm:ssZ`;
* no insignificant whitespace.

View File

@@ -1,190 +0,0 @@
# ARCHITECTURE.md — **StellaOps.Feedser**
> **Goal**: Build a sovereign-ready, self-hostable **feed-merge service** that ingests authoritative vulnerability sources, normalizes and de-duplicates them into **MongoDB**, and exports **JSON** and **Trivy-compatible DB** artifacts.
> **Form factor**: Long-running **Web Service** with **REST APIs** (health, status, control) and an embedded **internal cron scheduler**. Controllable by StellaOps.Cli (# stella db ...)
> **No signing inside Feedser** (signing is a separate pipeline step).
> **Runtime SDK baseline**: .NET 10 Preview 7 (SDK 10.0.100-preview.7.25380.108) targeting `net10.0`, aligned with the deployed api.stella-ops.org service.
> **Four explicit stages**:
>
> 1. **Source Download** → raw documents.
> 2. **Parse & Normalize** → schema-validated DTOs enriched with canonical identifiers.
> 3. **Merge & Deduplicate** → precedence-aware canonical records persisted to MongoDB.
> 4. **Export** → JSON or TrivyDB (full or delta), then (externally) sign/publish.
---
## 1) Naming & Solution Layout
**Source connectors** namespace prefix: `StellaOps.Feedser.Source.*`
**Exporters**:
* `StellaOps.Feedser.Exporter.Json`
* `StellaOps.Feedser.Exporter.TrivyDb`
**Projects** (`/src`):
```
StellaOps.Feedser.WebService/ # ASP.NET Core (Minimal API, net10.0 preview) WebService + embedded scheduler
StellaOps.Feedser.Core/ # Domain models, pipelines, merge/dedupe engine, jobs orchestration
StellaOps.Feedser.Models/ # Canonical POCOs, JSON Schemas, enums
StellaOps.Feedser.Storage.Mongo/ # Mongo repositories, GridFS access, indexes, resume "flags"
StellaOps.Feedser.Source.Common/ # HTTP clients, rate-limiters, schema validators, parsers utils
StellaOps.Feedser.Source.Cve/
StellaOps.Feedser.Source.Nvd/
StellaOps.Feedser.Source.Ghsa/
StellaOps.Feedser.Source.Osv/
StellaOps.Feedser.Source.Jvn/
StellaOps.Feedser.Source.CertCc/
StellaOps.Feedser.Source.Kev/
StellaOps.Feedser.Source.Kisa/
StellaOps.Feedser.Source.CertIn/
StellaOps.Feedser.Source.CertFr/
StellaOps.Feedser.Source.CertBund/
StellaOps.Feedser.Source.Acsc/
StellaOps.Feedser.Source.Cccs/
StellaOps.Feedser.Source.Ru.Bdu/ # HTML→schema with LLM fallback (gated)
StellaOps.Feedser.Source.Ru.Nkcki/ # PDF/HTML bulletins → structured
StellaOps.Feedser.Source.Vndr.Msrc/
StellaOps.Feedser.Source.Vndr.Cisco/
StellaOps.Feedser.Source.Vndr.Oracle/
StellaOps.Feedser.Source.Vndr.Adobe/ # APSB ingest; emits vendor RangePrimitives with adobe.track/platform/priority telemetry + fixed-status provenance.
StellaOps.Feedser.Source.Vndr.Apple/
StellaOps.Feedser.Source.Vndr.Chromium/
StellaOps.Feedser.Source.Vndr.Vmware/
StellaOps.Feedser.Source.Distro.RedHat/
StellaOps.Feedser.Source.Distro.Debian/ # Fetches DSA list + detail HTML, emits EVR RangePrimitives with per-release provenance and telemetry.
StellaOps.Feedser.Source.Distro.Ubuntu/ # Ubuntu Security Notices connector (JSON index → EVR ranges with ubuntu.pocket telemetry).
StellaOps.Feedser.Source.Distro.Suse/ # CSAF fetch pipeline emitting NEVRA RangePrimitives with suse.status vendor telemetry.
StellaOps.Feedser.Source.Ics.Cisa/
StellaOps.Feedser.Source.Ics.Kaspersky/
StellaOps.Feedser.Normalization/ # Canonical mappers, validators, version-range normalization
StellaOps.Feedser.Merge/ # Identity graph, precedence, deterministic merge
StellaOps.Feedser.Exporter.Json/
StellaOps.Feedser.Exporter.TrivyDb/
StellaOps.Feedser.<Component>.Tests/ # Component-scoped unit/integration suites (Core, Storage.Mongo, Source.*, Exporter.*, WebService, etc.)
```
---
## 2) Runtime Shape
**Process**: single service (`StellaOps.Feedser.WebService`)
* `Program.cs`: top-level entry using **Generic Host**, **DI**, **Options** binding from `appsettings.json` + environment + optional `feedser.yaml`.
* Built-in **scheduler** (cron-like) + **job manager** with **distributed locks** in Mongo to prevent overlaps, enforce timeouts, allow cancel/kill.
* **REST APIs** for health/readiness/progress/trigger/kill/status.
**Key NuGet concepts** (indicative): `MongoDB.Driver`, `Polly` (retry/backoff), `System.Threading.Channels`, `Microsoft.Extensions.Http`, `Microsoft.Extensions.Hosting`, `Serilog`, `OpenTelemetry`.
---
## 3) Data Storage — **MongoDB** (single source of truth)
**Database**: `feedser`
**Write concern**: `majority` for merge/export state, `acknowledged` for raw docs.
**Collections** (with “flags”/resume points):
* `source`
* `_id`, `name`, `type`, `baseUrl`, `auth`, `notes`.
* `source_state`
* Keys: `sourceName` (unique), `enabled`, `cursor`, `lastSuccess`, `failCount`, `backoffUntil`, `paceOverrides`, `paused`.
* Drives incremental fetch/parse/map resume and operator pause/pace controls.
* `document`
* `_id`, `sourceName`, `uri`, `fetchedAt`, `sha256`, `contentType`, `status`, `metadata`, `gridFsId`, `etag`, `lastModified`.
* Index `{sourceName:1, uri:1}` unique; optional TTL for superseded versions.
* `dto`
* `_id`, `sourceName`, `documentId`, `schemaVer`, `payload` (BSON), `validatedAt`.
* Index `{sourceName:1, documentId:1}`.
* `advisory`
* `_id`, `advisoryKey`, `title`, `summary`, `lang`, `published`, `modified`, `severity`, `exploitKnown`.
* Unique `{advisoryKey:1}` plus indexes on `modified` and `published`.
* `alias`
* `advisoryId`, `scheme`, `value` with index `{scheme:1, value:1}`.
* `affected`
* `advisoryId`, `platform`, `name`, `versionRange`, `cpe`, `purl`, `fixedBy`, `introducedVersion`.
* Index `{platform:1, name:1}`, `{advisoryId:1}`.
* `reference`
* `advisoryId`, `url`, `kind`, `sourceTag` (e.g., advisory/patch/kb).
* Flags collections: `kev_flag`, `ru_flags`, `jp_flags`, `psirt_flags` keyed by `advisoryId`.
* `merge_event`
* `_id`, `advisoryKey`, `beforeHash`, `afterHash`, `mergedAt`, `inputs` (document ids).
* `export_state`
* `_id` (`json`/`trivydb`), `baseExportId`, `baseDigest`, `lastFullDigest`, `lastDeltaDigest`, `exportCursor`, `targetRepo`, `exporterVersion`.
* `locks`
* `_id` (`jobKey`), `holder`, `acquiredAt`, `heartbeatAt`, `leaseMs`, `ttlAt` (TTL index cleans dead locks).
* `jobs`
* `_id`, `type`, `args`, `state`, `startedAt`, `endedAt`, `error`, `owner`, `heartbeatAt`, `timeoutMs`.
**GridFS buckets**: `fs.documents` for raw large payloads; referenced by `document.gridFsId`.
---
## 4) Job & Scheduler Model
* Scheduler stores cron expressions per source/exporter in config; persists next-run pointers in Mongo.
* Jobs acquire locks (`locks` collection) to ensure singleton execution per source/exporter.
* Supports manual triggers via API endpoints (`POST /jobs/{type}`) and pause/resume toggles per source.
---
## 5) Connector Contracts
Connectors implement:
```csharp
public interface IFeedConnector {
string SourceName { get; }
Task FetchAsync(IServiceProvider sp, CancellationToken ct);
Task ParseAsync(IServiceProvider sp, CancellationToken ct);
Task MapAsync(IServiceProvider sp, CancellationToken ct);
}
```
* Fetch populates `document` rows respecting rate limits, conditional GET, and `source_state.cursor`.
* Parse validates schema (JSON Schema, XSD) and writes sanitized DTO payloads.
* Map produces canonical advisory rows + provenance entries; must be idempotent.
* Base helpers in `StellaOps.Feedser.Source.Common` provide HTTP clients, retry policies, and watermark utilities.
---
## 6) Merge & Normalization
* Canonical model stored in `StellaOps.Feedser.Models` with serialization contracts used by storage/export layers.
* `StellaOps.Feedser.Normalization` handles NEVRA/EVR/PURL range parsing, CVSS normalization, localization.
* `StellaOps.Feedser.Merge` builds alias graphs keyed by CVE first, then falls back to vendor/regional IDs.
* Precedence rules: PSIRT/OVAL overrides generic ranges; KEV only toggles exploitation; regional feeds enrich severity but dont override vendor truth.
* Determinism enforced via canonical JSON hashing logged in `merge_event`.
---
## 7) Exporters
* JSON exporter mirrors `aquasecurity/vuln-list` layout with deterministic ordering and reproducible timestamps.
* Trivy DB exporter shells out to `trivy-db build`, produces Bolt archives, and reuses unchanged blobs from the last full baseline when running in delta mode. The exporter annotates `metadata.json` with `mode`, `baseExportId`, `baseManifestDigest`, `resetBaseline`, and `delta.changedFiles[]`/`delta.removedPaths[]`, and honours `publishFull` / `publishDelta` (ORAS) plus `includeFull` / `includeDelta` (offline bundle) toggles.
* `StellaOps.Feedser.Storage.Mongo` provides cursors for delta exports based on `export_state.exportCursor` and the persisted per-file manifest (`export_state.files`).
* Export jobs produce OCI tarballs (layer media type `application/vnd.aquasec.trivy.db.layer.v1.tar+gzip`) and optionally push via ORAS; `metadata.json` accompanies each layout so mirrors can decide between full refreshes and deltas.
---
## 8) Observability
* Serilog structured logging with enrichment fields (`source`, `uri`, `stage`, `durationMs`).
* OpenTelemetry traces around fetch/parse/map/export; metrics for rate limit hits, schema failures, dedupe ratios, package size. Connector HTTP metrics are emitted via the shared `feedser.source.http.*` instruments tagged with `feedser.source=<connector>` so per-source dashboards slice on that label instead of bespoke metric names.
* Prometheus scraping endpoint served by WebService.
---
## 9) Security Considerations
* Offline-first: connectors only reach allowlisted hosts.
* BDU LLM fallback gated by config flag; logs audit trail with confidence score.
* No secrets written to logs; secrets loaded via environment or mounted files.
* Signing handled outside Feedser pipeline.
---
## 10) Deployment Notes
* Default storage MongoDB; for air-gapped, bundle Mongo image + seeded data backup.
* Horizontal scale achieved via multiple web service instances sharing Mongo locks.
* Provide `feedser.yaml` template describing sources, rate limits, and export settings.

View File

@@ -0,0 +1,413 @@
# component_architecture_scanner.md — **StellaOps Scanner** (2025Q4)
> **Scope.** Implementationready architecture for the **Scanner** subsystem: WebService, Workers, analyzers, SBOM assembly (inventory & usage), perlayer caching, threeway diffs, artifact catalog (MinIO+Mongo), attestation handoff, and scale/security posture. This document is the contract between the scanning plane and everything else (Policy, Excititor, Concelier, UI, CLI).
---
## 0) Mission & boundaries
**Mission.** Produce **deterministic**, **explainable** SBOMs and diffs for container images and filesystems, quickly and repeatedly, without guessing. Emit two views: **Inventory** (everything present) and **Usage** (entrypoint closure + actually linked libs). Attach attestations through **Signer→Attestor→Rekor v2**.
**Boundaries.**
* Scanner **does not** produce PASS/FAIL. The backend (Policy + Excititor + Concelier) decides presentation and verdicts.
* Scanner **does not** keep thirdparty SBOM warehouses. It may **bind** to existing attestations for exact hashes.
* Core analyzers are **deterministic** (no fuzzy identity). Optional heuristic plugins (e.g., patchpresence) run under explicit flags and never contaminate the core SBOM.
---
## 1) Solution & project layout
```
src/
├─ StellaOps.Scanner.WebService/ # REST control plane, catalog, diff, exports
├─ StellaOps.Scanner.Worker/ # queue consumer; executes analyzers
├─ StellaOps.Scanner.Models/ # DTOs, evidence, graph nodes, CDX/SPDX adapters
├─ StellaOps.Scanner.Storage/ # Mongo repositories; MinIO object client; ILM/GC
├─ StellaOps.Scanner.Queue/ # queue abstraction (Redis/NATS/RabbitMQ)
├─ StellaOps.Scanner.Cache/ # layer cache; file CAS; bloom/bitmap indexes
├─ StellaOps.Scanner.EntryTrace/ # ENTRYPOINT/CMD → terminal program resolver (shell AST)
├─ StellaOps.Scanner.Analyzers.OS.[Apk|Dpkg|Rpm]/
├─ StellaOps.Scanner.Analyzers.Lang.[Java|Node|Python|Go|DotNet|Rust]/
├─ StellaOps.Scanner.Analyzers.Native.[ELF|PE|MachO]/ # PE/Mach-O planned (M2)
├─ StellaOps.Scanner.Emit.CDX/ # CycloneDX (JSON + Protobuf)
├─ StellaOps.Scanner.Emit.SPDX/ # SPDX 3.0.1 JSON
├─ StellaOps.Scanner.Diff/ # image→layer→component threeway diff
├─ StellaOps.Scanner.Index/ # BOMIndex sidecar (purls + roaring bitmaps)
├─ StellaOps.Scanner.Tests.* # unit/integration/e2e fixtures
└─ tools/
├─ StellaOps.Scanner.Sbomer.BuildXPlugin/ # BuildKit generator (image referrer SBOMs)
└─ StellaOps.Scanner.Sbomer.DockerImage/ # CLIdriven scanner container
```
**Runtime formfactor:** two deployables
* **Scanner.WebService** (stateless REST)
* **Scanner.Worker** (N replicas; queuedriven)
---
## 2) External dependencies
* **OCI registry** with **Referrers API** (discover attached SBOMs/signatures).
* **MinIO** (S3compatible) for SBOM artifacts; **Object Lock** for immutable classes; **ILM** for TTL.
* **MongoDB** for catalog, job state, diffs, ILM rules.
* **Queue** (Redis Streams/NATS/RabbitMQ).
* **Authority** (onprem OIDC) for **OpToks** (DPoP/mTLS).
* **Signer** + **Attestor** (+ **Fulcio/KMS** + **Rekor v2**) for DSSE + transparency.
---
## 3) Contracts & data model
### 3.1 Evidencefirst component model
**Nodes**
* `Image`, `Layer`, `File`
* `Component` (`purl?`, `name`, `version?`, `type`, `id` — may be `bin:{sha256}`)
* `Executable` (ELF/PE/MachO), `Library` (native or managed), `EntryScript` (shell/launcher)
**Edges** (all carry **Evidence**)
* `contains(Image|Layer → File)`
* `installs(PackageDB → Component)` (OS database row)
* `declares(InstalledMetadata → Component)` (distinfo, pom.properties, deps.json…)
* `links_to(Executable → Library)` (ELF `DT_NEEDED`, PE imports)
* `calls(EntryScript → Program)` (file:line from shell AST)
* `attests(Rekor → Component|Image)` (SBOM/predicate binding)
* `bound_from_attestation(Component_attested → Component_observed)` (hash equality proof)
**Evidence**
```
{ source: enum, locator: (path|offset|line), sha256?, method: enum, timestamp }
```
No confidences. Either a fact is proven with listed mechanisms, or it is not claimed.
### 3.2 Catalog schema (Mongo)
* `artifacts`
```
{ _id, type: layer-bom|image-bom|diff|index,
format: cdx-json|cdx-pb|spdx-json,
bytesSha256, size, rekor: { uuid,index,url }?,
ttlClass, immutable, refCount, createdAt }
```
* `images { imageDigest, repo, tag?, arch, createdAt, lastSeen }`
* `layers { layerDigest, mediaType, size, createdAt, lastSeen }`
* `links { fromType, fromDigest, artifactId }` // image/layer -> artifact
* `jobs { _id, kind, args, state, startedAt, heartbeatAt, endedAt, error }`
* `lifecycleRules { ruleId, scope, ttlDays, retainIfReferenced, immutable }`
### 3.3 Object store layout (MinIO)
```
layers/<sha256>/sbom.cdx.json.zst
layers/<sha256>/sbom.spdx.json.zst
images/<imgDigest>/inventory.cdx.pb # CycloneDX Protobuf
images/<imgDigest>/usage.cdx.pb
indexes/<imgDigest>/bom-index.bin # purls + roaring bitmaps
diffs/<old>_<new>/diff.json.zst
attest/<artifactSha256>.dsse.json # DSSE bundle (cert chain + Rekor proof)
```
---
## 4) REST API (Scanner.WebService)
All under `/api/v1/scanner`. Auth: **OpTok** (DPoP/mTLS); RBAC scopes.
```
POST /scans { imageRef|digest, force?:bool } → { scanId }
GET /scans/{id} → { status, imageDigest, artifacts[], rekor? }
GET /sboms/{imageDigest} ?format=cdx-json|cdx-pb|spdx-json&view=inventory|usage → bytes
GET /diff?old=<digest>&new=<digest>&view=inventory|usage → diff.json
POST /exports { imageDigest, format, view, attest?:bool } → { artifactId, rekor? }
POST /reports { imageDigest, policyRevision? } → { reportId, rekor? } # delegates to backend policy+vex
GET /catalog/artifacts/{id} → { meta }
GET /healthz | /readyz | /metrics
```
---
## 5) Execution flow (Worker)
### 5.1 Acquire & verify
1. **Resolve image** (prefer `repo@sha256:`).
2. **(Optional) verify image signature** per policy (cosign).
3. **Pull blobs**, compute layer digests; record metadata.
### 5.2 Layer union FS
* Apply whiteouts; materialize final filesystem; map **file → first introducing layer**.
* Windows layers (MSI/SxS/GAC) planned in **M2**.
### 5.3 Evidence harvest (parallel analyzers; deterministic only)
**A) OS packages**
* **apk**: `/lib/apk/db/installed`
* **dpkg**: `/var/lib/dpkg/status`, `/var/lib/dpkg/info/*.list`
* **rpm**: `/var/lib/rpm/Packages` (via librpm or parser)
* Record `name`, `version` (epoch/revision), `arch`, source package where present, and **declared file lists**.
**B) Language ecosystems (installed state only)**
* **Java**: `META-INF/maven/*/pom.properties`, MANIFEST → `pkg:maven/...`
* **Node**: `node_modules/**/package.json` → `pkg:npm/...`
* **Python**: `*.dist-info/{METADATA,RECORD}` → `pkg:pypi/...`
* **Go**: Go **buildinfo** in binaries → `pkg:golang/...`
* **.NET**: `*.deps.json` + assembly metadata → `pkg:nuget/...`
* **Rust**: crates only when **explicitly present** (embedded metadata or cargo/registry traces); otherwise binaries reported as `bin:{sha256}`.
> **Rule:** We only report components proven **on disk** with authoritative metadata. Lockfiles are evidence only.
**C) Native link graph**
* **ELF**: parse `PT_INTERP`, `DT_NEEDED`, RPATH/RUNPATH, **GNU symbol versions**; map **SONAMEs** to file paths; link executables → libs.
* **PE/MachO** (planned M2): import table, delayimports; version resources; code signatures.
* Map libs back to **OS packages** if possible (via file lists); else emit `bin:{sha256}` components.
**D) EntryTrace (ENTRYPOINT/CMD → terminal program)**
* Read image config; parse shell (POSIX/Bash subset) with AST: `source`/`.` includes; `case/if`; `exec`/`command`; `runparts`.
* Resolve commands via **PATH** within the **built rootfs**; follow language launchers (Java/Node/Python) to identify the terminal program (ELF/JAR/venv script).
* Record **file:line** and choices for each hop; output chain graph.
* Unresolvable dynamic constructs are recorded as **unknown** edges with reasons (e.g., `$FOO` unresolved).
**E) Attestation & SBOM bind (optional)**
* For each **file hash** or **binary hash**, query local cache of **Rekor v2** indices; if an SBOM attestation is found for **exact hash**, bind it to the component (origin=`attested`).
* For the **image** digest, likewise bind SBOM attestations (buildtime referrers).
### 5.4 Component normalization (exact only)
* Create `Component` nodes only with deterministic identities: purl, or **`bin:{sha256}`** for unlabeled binaries.
* Record **origin** (OS DB, installed metadata, linker, attestation).
### 5.5 SBOM assembly & emit
* **Perlayer SBOM fragments**: components introduced by the layer (+ relationships).
* **Image SBOMs**: merge fragments; refer back to them via **CycloneDX BOMLink** (or SPDX ExternalRef).
* Emit both **Inventory** & **Usage** views.
* Serialize **CycloneDX JSON** and **CycloneDX Protobuf**; optionally **SPDX 3.0.1 JSON**.
* Build **BOMIndex** sidecar: purl table + roaring bitmap; flag `usedByEntrypoint` components for fast backend joins.
### 5.6 DSSE attestation (via Signer/Attestor)
* WebService constructs **predicate** with `image_digest`, `stellaops_version`, `license_id`, `policy_digest?` (when emitting **final reports**), timestamps.
* Calls **Signer** (requires **OpTok + PoE**); Signer verifies **entitlement + scanner image integrity** and returns **DSSE bundle**.
* **Attestor** logs to **Rekor v2**; returns `{uuid,index,proof}` → stored in `artifacts.rekor`.
---
## 6) Threeway diff (image → layer → component)
### 6.1 Keys & classification
* Component key: **purl** when present; else `bin:{sha256}`.
* Diff classes: `added`, `removed`, `version_changed` (`upgraded|downgraded`), `metadata_changed` (e.g., origin from attestation vs observed).
* Layer attribution: for each change, resolve the **introducing/removing layer**.
### 6.2 Algorithm (outline)
```
A = components(imageOld, key)
B = components(imageNew, key)
added = B \ A
removed = A \ B
changed = { k in A∩B : version(A[k]) != version(B[k]) || origin changed }
for each item in added/removed/changed:
layer = attribute_to_layer(item, imageOld|imageNew)
usageFlag = usedByEntrypoint(item, imageNew)
emit diff.json (grouped by layer with badges)
```
Diffs are stored as artifacts and feed **UI** and **CLI**.
---
## 7) Buildtime SBOMs (fast CI path)
**Scanner.Sbomer.BuildXPlugin** can act as a BuildKit **generator**:
* During `docker buildx build --attest=type=sbom,generator=stellaops/sbom-indexer`, run analyzers on the build context/output; attach SBOMs as OCI **referrers** to the built image.
* Optionally request **Signer/Attestor** to produce **StellaOpsverified** attestation immediately; else, Scanner.WebService can verify and reattest postpush.
* Scanner.WebService trusts buildtime SBOMs per policy, enabling **norescan** for unchanged bases.
---
## 8) Configuration (YAML)
```yaml
scanner:
queue:
kind: redis
url: "redis://queue:6379/0"
mongo:
uri: "mongodb://mongo/scanner"
s3:
endpoint: "http://minio:9000"
bucket: "stellaops"
objectLock: "governance" # or 'compliance'
analyzers:
os: { apk: true, dpkg: true, rpm: true }
lang: { java: true, node: true, python: true, go: true, dotnet: true, rust: true }
native: { elf: true, pe: false, macho: false } # PE/Mach-O in M2
entryTrace: { enabled: true, shellMaxDepth: 64, followRunParts: true }
emit:
cdx: { json: true, protobuf: true }
spdx: { json: true }
compress: "zstd"
rekor:
url: "https://rekor-v2.internal"
signer:
url: "https://signer.internal"
limits:
maxParallel: 8
perRegistryConcurrency: 2
policyHints:
verifyImageSignature: false
trustBuildTimeSboms: true
```
---
## 9) Scale & performance
* **Parallelism**: peranalyzer concurrency; bounded directory walkers; file CAS dedupe by sha256.
* **Distributed locks** per **layer digest** to prevent duplicate work across Workers.
* **Registry throttles**: perhost concurrency budgets; exponential backoff on 429/5xx.
* **Targets**:
* **Buildtime**: P95 ≤35s on warmed bases (CI generator).
* **Postbuild delta**: P95 ≤10s for 200MB images with cache hit.
* **Emit**: CycloneDX Protobuf ≤150ms for 5k components; JSON ≤500ms.
* **Diff**: ≤200ms for 5k vs 5k components.
---
## 10) Security posture
* **AuthN**: Authorityissued short OpToks (DPoP/mTLS).
* **AuthZ**: scopes (`scanner.scan`, `scanner.export`, `scanner.catalog.read`).
* **mTLS** to **Signer**/**Attestor**; only **Signer** can sign.
* **No network fetches** during analysis (except registry pulls and optional Rekor index reads).
* **Sandboxing**: nonroot containers; readonly FS; seccomp profiles; disable execution of scanned content.
* **Release integrity**: all firstparty images are **cosignsigned**; Workers/WebService selfverify at startup.
---
## 11) Observability & audit
* **Metrics**:
* `scanner.jobs_inflight`, `scanner.scan_latency_seconds`
* `scanner.layer_cache_hits_total`, `scanner.file_cas_hits_total`
* `scanner.artifact_bytes_total{format}`
* `scanner.attestation_latency_seconds`, `scanner.rekor_failures_total`
* **Tracing**: spans for acquire→union→analyzers→compose→emit→sign→log.
* **Audit logs**: DSSE requests log `license_id`, `image_digest`, `artifactSha256`, `policy_digest?`, Rekor UUID on success.
---
## 12) Testing matrix
* **Determinism:** given same image + analyzers → byteidentical **CDX Protobuf**; JSON normalized.
* **OS packages:** groundtruth images per distro; compare to package DB.
* **Lang ecosystems:** sample images per ecosystem (Java/Node/Python/Go/.NET/Rust) with installed metadata; negative tests w/ lockfileonly.
* **Native & EntryTrace:** ELF graph correctness; shell AST cases (includes, runparts, exec, case/if).
* **Diff:** layer attribution against synthetic twoimage sequences.
* **Performance:** cold vs warm cache; large `node_modules` and `sitepackages`.
* **Security:** ensure no code execution from image; fuzz parser inputs; path traversal resistance on layer extract.
---
## 13) Failure modes & degradations
* **Missing OS DB** (files exist, DB removed): record **files**; do **not** fabricate package components; emit `bin:{sha256}` where unavoidable; flag in evidence.
* **Unreadable metadata** (corrupt distinfo): record file evidence; skip component creation; annotate.
* **Dynamic shell constructs**: mark unresolved edges with reasons (env var unknown) and continue; **Usage** view may be partial.
* **Registry rate limits**: honor backoff; queue job retries with jitter.
* **Signer refusal** (license/plan/version): scan completes; artifact produced; **no attestation**; WebService marks result as **unverified**.
---
## 14) Optional plugins (off by default)
* **Patchpresence detector** (signaturebased backport checks). Reads curated functionlevel signatures from advisories; inspects binaries for patched code snippets to lower falsepositives for backported fixes. Runs as a sidecar analyzer that **annotates** components; never overrides core identities.
* **Runtime probes** (with Zastava): when allowed, compare **/proc/<pid>/maps** (DSOs actually loaded) with static **Usage** view for precision.
---
## 15) DevOps & operations
* **HA**: WebService horizontal scale; Workers autoscale by queue depth & CPU; distributed locks on layers.
* **Retention**: ILM rules per artifact class (`short`, `default`, `compliance`); **Object Lock** for compliance artifacts (reports, signed SBOMs).
* **Upgrades**: bump **cache schema** when analyzer outputs change; WebService triggers refresh of dependent artifacts.
* **Backups**: Mongo (daily dumps); MinIO (versioned buckets, replication); Rekor v2 DB snapshots.
---
## 16) CLI & UI touch points
* **CLI**: `stellaops scan <ref>`, `stellaops diff --old --new`, `stellaops export`, `stellaops verify attestation <bundle|url>`.
* **UI**: Scan detail shows **Inventory/Usage** toggles, **Diff by Layer**, **Attestation badge** (verified/unverified), Rekor link, and **EntryTrace** chain with file:line breadcrumbs.
---
## 17) Roadmap (Scanner)
* **M2**: Windows containers (MSI/SxS/GAC analyzers), PE/MachO native analyzer, deeper Rust metadata.
* **M2**: Buildx generator GA (certified external registries), crossregistry trust policies.
* **M3**: Patchpresence plugin GA (optin), crossimage corpus clustering (evidenceonly; not identity).
* **M3**: Advanced EntryTrace (POSIX shell features breadth, busybox detection).
---
### Appendix A — EntryTrace resolution (pseudo)
```csharp
ResolveEntrypoint(ImageConfig cfg, RootFs fs):
cmd = Normalize(cfg.ENTRYPOINT, cfg.CMD)
stack = [ Script(cmd, path=FindOnPath(cmd[0], fs)) ]
visited = set()
while stack not empty and depth < MAX:
cur = stack.pop()
if cur in visited: continue
visited.add(cur)
if IsShellScript(cur.path):
ast = ParseShell(cur.path)
foreach directive in ast:
if directive is Source include:
p = ResolveInclude(include.path, cur.env, fs)
stack.push(Script(p))
if directive is Exec call:
p = ResolveExec(call.argv[0], cur.env, fs)
stack.push(Program(p, argv=call.argv))
if directive is Interpreter (python -m / node / java -jar):
term = ResolveInterpreterTarget(call, fs)
stack.push(Program(term))
else:
return Terminal(cur.path)
return Unknown(reason)
```
### Appendix B — BOMIndex sidecar
```
struct Header { magic, version, imageDigest, createdAt }
vector<string> purls
map<purlIndex, roaring_bitmap> components
optional map<purlIndex, roaring_bitmap> usedByEntrypoint
```

418
docs/ARCHITECTURE_SIGNER.md Normal file
View File

@@ -0,0 +1,418 @@
# component_architecture_signer.md — **StellaOps Signer** (2025Q4)
> **Scope.** Implementationready architecture for the **Signer**: the *only* service allowed to produce **StellaOpsverified** signatures over SBOMs and reports. It enforces **entitlement** (PoE), **release integrity** (scanner provenance), **senderconstrained auth** (DPoP/mTLS), and emits **intoto/DSSE** bundles suitable for **Rekor v2** logging by the Attestor. Includes APIs, data flow, storage, quotas, security, and test matrices.
---
## 0) Mission & boundaries
**Mission.** Convert authenticated signing requests from trusted StellaOps services into **verifiable** DSSE bundles while enforcing **license policy** and **supplychain integrity**.
**Boundaries.**
* **Signer does not push to Rekor** — it returns DSSE to the caller; **Attestor** logs to **Rekor v2**.
* **Signer does not compute PASS/FAIL** — it signs SBOMs/reports produced by Scanner/WebService after backend evaluation.
* **Signer is stateless for hot path** — longterm storage is limited to audit events; all secrets/keys live in KMS/HSM or are ephemeral (keyless).
---
## 1) Responsibilities (contract)
1. **Authenticate** caller with **OpTok** (Authority OIDC, DPoP or mTLSbound).
2. **Authorize** scopes (`signer.sign`) + audience (`aud=signer`) + tenant/installation.
3. **Validate entitlement** via **PoE** (ProofofEntitlement) against Cloud Licensing `/license/introspect`.
4. **Verify release integrity** of the **scanner** image digest presented in the request: must be **cosignsigned** by StellaOps release key, discoverable via **OCI Referrers API**.
5. **Enforce plan & quotas** (concurrency/QPS/artifact size/rate caps).
6. **Mint signing identity**:
* **Keyless** (default): get a shortlived X.509 cert from **Fulcio** using the Signers OIDC identity and sign the DSSE.
* **Keyful** (optional): sign with an HSM/KMS key.
7. **Return DSSE bundle** (subject digests + predicate + cert chain or KMS key id).
8. **Audit** every decision; expose metrics.
---
## 2) External dependencies
* **Authority** (onprem OIDC): validates OpToks (JWKS/introspection) and DPoP/mTLS.
* **Licensing Service (cloud)**: `/license/introspect` to verify PoE (active, claims, expiry, revocation).
* **Fulcio** (Sigstore) *or* **KMS/HSM**: to obtain certs or perform signatures.
* **OCI Registry (Referrers API)**: to verify **scanner** image release signature.
* **Attestor**: downstream service that writes DSSE bundles to **Rekor v2**.
* **Config/state stores**: Redis (caches, rate buckets), Mongo/Postgres (audit log).
---
## 3) API surface (mTLS; DPoP supported)
Base path: `/api/v1/signer`. **All endpoints require**:
* Access token (JWT) from **Authority** with `aud=signer`, `scope=signer.sign`.
* **Sender constraint**: DPoP proof per request or mTLS client cert.
* **PoE** presented as either:
* **Client TLS cert** (if PoE is mTLSstyle) chained to Licensing CA, *or*
* **PoE JWT** (DPoP/mTLSbound) in `X-PoE` header or request body.
### 3.1 `POST /sign/dsse`
Request (JSON):
```json
{
"subject": [
{ "name": "s3://stellaops/images/sha256:.../inventory.cdx.pb",
"digest": { "sha256": "..." } }
],
"predicateType": "https://stella-ops.org/attestations/sbom/1",
"predicate": {
"image_digest": "sha256:...",
"stellaops_version": "2.3.1 (2027.04)",
"license_id": "LIC-9F2A...",
"customer_id": "CUST-ACME",
"plan": "pro",
"policy_digest": "sha256:...", // optional for final reports
"views": ["inventory", "usage"],
"created": "2025-10-17T12:34:56Z"
},
"scannerImageDigest": "sha256:sc-web-or-worker-digest",
"poe": {
"format": "jwt", // or "mtls"
"value": "eyJhbGciOi..." // PoE JWT when not using mTLS PoE
},
"options": {
"signingMode": "keyless", // "keyless" | "kms"
"expirySeconds": 600, // cert lifetime hint (keyless)
"returnBundle": "dsse+cert" // dsse (default) | dsse+cert
}
}
```
Response 200:
```json
{
"bundle": {
"dsse": { "payloadType": "application/vnd.in-toto+json", "payload": "<base64>", "signatures": [ ... ] },
"certificateChain": [ "-----BEGIN CERTIFICATE-----...", "... root ..." ],
"mode": "keyless",
"signingIdentity": { "issuer": "https://fulcio.internal", "san": "urn:stellaops:signer", "certExpiry": "2025-10-17T12:44:56Z" }
},
"policy": { "plan": "pro", "maxArtifactBytes": 104857600, "qpsRemaining": 97 },
"auditId": "a7c9e3f2-1b7a-4e87-8c3a-90d7d2c3ad12"
}
```
Errors (RFC7807):
* `401 invalid_token` (JWT/DPoP/mTLS failure)
* `403 entitlement_denied` (PoE invalid/revoked/expired; release year mismatch)
* `403 release_untrusted` (scanner image not Stellasigned)
* `429 plan_throttled` (license plan caps)
* `413 artifact_too_large` (size cap)
* `400 invalid_request` (schema/predicate/type invalid)
* `500 signing_unavailable` (Fulcio/KMS outage)
### 3.2 `GET /verify/referrers?imageDigest=<sha256>`
Checks whether the **image** at digest is signed by **StellaOps release key**.
Response:
```json
{ "trusted": true, "signatures": [ { "type": "cosign", "digest": "sha256:...", "signedBy": "StellaOps Release 2027 Q2" } ] }
```
> **Note:** This endpoint is also used internally by Signer before issuing signatures.
---
## 4) Validation pipeline (hot path)
```mermaid
sequenceDiagram
autonumber
participant Client as Scanner.WebService
participant Auth as Authority (OIDC)
participant Sign as Signer
participant Lic as Licensing Service (cloud)
participant Reg as OCI Registry (Referrers)
participant Ful as Fulcio/KMS
Client->>Sign: POST /sign/dsse (OpTok + DPoP/mTLS, PoE, request)
Note over Sign: 1) Validate OpTok, audience, scope, DPoP/mTLS binding
Sign->>Lic: /license/introspect(PoE)
Lic-->>Sign: { active, claims: {license_id, plan, valid_release_year, max_version}, exp }
Note over Sign: 2) Enforce plan/version window and revocation
Sign->>Reg: Verify scannerImageDigest signed (Referrers + cosign)
Reg-->>Sign: OK with signer identity
Note over Sign: 3) Enforce release integrity
Note over Sign: 4) Enforce quotas (QPS/concurrency/size)
Sign->>Ful: Mint cert (keyless) or sign via KMS
Ful-->>Sign: Cert or signature
Sign-->>Client: DSSE bundle (+cert chain), policy counters, auditId
```
**DPoP nonce dance (when enabled for highvalue ops):**
* If DPoP proof lacks a valid nonce, Signer replies `401` with `WWW-Authenticate: DPoP error="use_dpop_nonce", dpop_nonce="<nonce>"`.
* Client retries with new proof including the nonce; Signer validates nonce and `jti` uniqueness (Redis TTL cache).
---
## 5) Entitlement enforcement (PoE)
* **Accepted forms**:
* **mTLS PoE**: client presents a **PoE client cert** at TLS handshake; Signer validates chain to **Licensing CA** (CA bundle configured) and calls `/license/introspect` with cert thumbprint + serial.
* **JWT PoE**: `X-PoE` bearer token (DPoP/mTLSbound) is validated (sig + `cnf`) locally (Licensing JWKS) and then **introspected** for status and claims.
* **Claims required**:
* `license_id`, `plan` (free|pro|enterprise|gov), `valid_release_year`, `max_version`, `exp`.
* Optional: `tenant_id`, `customer_id`, `entitlements[]`.
* **Enforcements**:
* Reject if **revoked**, **expired**, **plan mismatch** or **release outside window** (`stellaops_version` in predicate exceeds `max_version` or release date beyond `valid_release_year`).
* Apply plan **throttles** (QPS/concurrency/artifact bytes) via tokenbucket in Redis keyed by `license_id`.
---
## 6) Release integrity (scanner provenance)
* **Input**: `scannerImageDigest` representing the actual Scanner component that produced the artifact.
* **Check**:
1. Use **OCI Referrers API** to enumerate signatures of that digest.
2. Verify **cosign** signatures against the configured **StellaOps Release** keyring (keyless Fulcio roots *or* keyful public keys).
3. Optionally require Rekor inclusion for those signatures.
* **Policy**:
* If not signed by an authorized **StellaOps Release** identity → **deny**.
* If signed but **release year** > PoE `valid_release_year`**deny**.
* **Cache**: LRU of digest → verification result (TTL 1030min) to avoid registry thrash.
---
## 7) Signing modes
### 7.1 Keyless (default; Sigstore Fulcio)
* Signer authenticates to **Fulcio** using its onprem OIDC identity (client credentials) and requests a **shortlived cert** (510min).
* Generates **ephemeral keypair**, gets cert for the public key, signs DSSE with the **private key**.
* DSSE **bundle** includes **certificate chain**; verifiers validate to Fulcio root.
### 7.2 Keyful (optional; KMS/HSM)
* Signer uses a configured **KMS** key (AWS KMS, GCP KMS, Azure Key Vault, Vault Transit, or HSM).
* DSSE bundle includes **key metadata** (kid, cert chain if x509).
* Recommended for FIPS/sovereign environments.
---
## 8) Predicates & schema
Supported **predicate types** (extensible):
* `https://stella-ops.org/attestations/sbom/1` (SBOM emissions)
* `https://stella-ops.org/attestations/report/1` (final PASS/FAIL reports)
* `https://stella-ops.org/attestations/vex-export/1` (Excititor exports; optional)
**Validation**:
* JSONSchema per predicate type; **canonical property order**.
* `subject[*].digest` must include `sha256`.
* `predicate.stellaops_version` must parse and match policy windows.
---
## 9) Quotas & throttling
Per `license_id` (from PoE):
* **QPS** (token bucket), **concurrency** (semaphore), **artifact bytes** (sliding window).
* On exceed → `429 plan_throttled` with `Retry-After`.
* Free/community plan may also receive **randomized delay** to disincentivize farmed signing.
---
## 10) Storage & caches
* **Redis**:
* DPoP nonce & `jti` replay cache (TTL ≤ 10min).
* PoE introspection cache (short TTL, e.g., 60120s).
* Releaseverify cache (`scannerImageDigest` → { trusted, ts }).
* **Audit store** (Mongo or Postgres): `signer.audit_events`
```
{ _id, ts, tenantId, installationId, licenseId, customerId,
plan, actor{sub,cnf}, request{predicateType, subjectSha256[], imageDigest},
poe{type, thumbprint|jwtKid, exp, introspectSnapshot},
release{digest, signerId, policy},
mode: "keyless"|"kms",
result: "success"|"deny:<reason>"|"error:<reason>",
bundleSha256? }
```
* **Config**: StellaOps release signing keyring, Fulcio roots, Licensing CA bundle.
---
## 11) Security & privacy
* **mTLS** on all Signer endpoints.
* **No bearer fallbacks** — DPoP/mTLS enforced for `aud=signer`.
* **PoE** is never persisted beyond audit snapshots (minimized fields).
* **Secrets**: no longlived private keys on disk (keyless) or handled via KMS APIs.
* **Input hardening**: schemavalidate predicates; cap payload sizes; zstd/gzip decompression bombs guarded.
* **Logging**: redact PoE JWTs, access tokens, DPoP proofs; log only hashes and identifiers.
---
## 12) Metrics & observability
* `signer.requests_total{result}`
* `signer.latency_seconds{stage=auth|introspect|release_verify|sign}`
* `signer.poe_failures_total{reason}`
* `signer.release_verify_failures_total{reason}`
* `signer.plan_throttle_total{license_id}`
* `signer.bundle_bytes_total`
* `signer.keyless_certs_issued_total` / `signer.kms_sign_total`
* OTEL traces across stages; correlation id (`auditId`) returned to client.
---
## 13) Configuration (YAML)
```yaml
signer:
listen: "https://0.0.0.0:8443"
authority:
issuer: "https://authority.internal"
jwksUrl: "https://authority.internal/jwks"
require: "dpop" # "dpop" | "mtls"
poe:
mode: "both" # "jwt" | "mtls" | "both"
licensing:
introspectUrl: "https://www.stella-ops.org/api/v1/license/introspect"
caBundle: "/etc/ssl/licensing-ca.pem"
cacheTtlSeconds: 90
release:
referrers:
allowRekorVerified: true
keyrings:
- type: "cosign-keyless"
fulcioRoots: ["/etc/fulcio/root.pem"]
identities:
- san: "mailto:release@stella-ops.org"
- san: "https://sigstore.dev/oidc/stellaops"
signing:
mode: "keyless" # "keyless" | "kms"
fulcio:
issuer: "https://fulcio.internal"
oidcClientId: "signer"
oidcClientSecretRef: "env:FULCIO_CLIENT_SECRET"
certTtlSeconds: 600
kms:
provider: "aws-kms"
keyId: "arn:aws:kms:...:key/..."
quotas:
default:
qps: 100
concurrency: 20
maxArtifactBytes: 104857600
free:
qps: 5
concurrency: 1
maxArtifactBytes: 1048576
```
---
## 14) Testing matrix
* **Auth & DPoP**: bad `aud`, wrong `jkt`, replayed `jti`, missing nonce, mTLS mismatch.
* **PoE**: expired, revoked, plan mismatch, release year gate, max_version gate.
* **Release verify**: unsigned digest, wrong signer, Rekorabsent (when required), referrers unreachable.
* **Signing**: Fulcio outage; KMS timeouts; bundle correctness (verifier harness).
* **Quotas**: burst above QPS, artifact over size, concurrency overflow.
* **Schema**: invalid predicate types/required fields.
* **Determinism**: same request → identical DSSE (aside from cert validity period).
* **Perf**: P95 endtoend under 120ms with caches warm (excluding network to Fulcio).
---
## 15) Failure modes & responses
| Failure | HTTP | Problem type | Notes |
| ----------------------- | ---- | --------------------- | -------------------------------------------- |
| Invalid OpTok / DPoP | 401 | `invalid_token` | `WWW-Authenticate` with DPoP nonce if needed |
| PoE invalid/revoked | 403 | `entitlement_denied` | Include `license_id` (hashed) and reason |
| Scanner image untrusted | 403 | `release_untrusted` | Include digest and required identity |
| Plan throttle | 429 | `plan_throttled` | Include limits and `Retry-After` |
| Artifact too large | 413 | `artifact_too_large` | Include cap |
| Fulcio/KMS down | 503 | `signing_unavailable` | RetryAfter with jitter |
---
## 16) Deployment & HA
* Run ≥ 2 replicas; front with L7 LB; **sticky** not required.
* Redis for replay/quota caches (HA).
* Audit sink (Mongo/Postgres) in primary region; asynchronous write with local fallback buffer.
* Fulcio/KMS clients configured with retries/backoff; circuit breakers.
---
## 17) Implementation notes
* **.NET 10** minimal API + Kestrel mTLS; custom DPoP middleware; JWT/JWKS cache.
* **Cosign verification** via sigstore libraries; Referrers queries over registry API with retries.
* **DSSE** via intoto libs; canonical JSON writer for predicates.
* **Backpressure** paths: refuse at auth/quota stages before any expensive network calls.
---
## 18) Examples (wire)
**Request (free plan; expect throttle if burst):**
```http
POST /api/v1/signer/sign/dsse HTTP/1.1
Authorization: DPoP <JWT>
DPoP: <proof>
Content-Type: application/json
```
**Error (release untrusted):**
```json
{
"type": "https://stella-ops.org/problems/release_untrusted",
"title": "Scanner image not signed by StellaOps",
"status": 403,
"detail": "sha256:abcd... not in trusted keyring",
"instance": "urn:audit:a7c9e3f2-..."
}
```
---
## 19) Roadmap
* **Key Transparency**: optional publication of Signers *own* certs to a KT log.
* **Attested Build**: SLSAstyle provenance for Signer container itself, checked at startup.
* **FIPS mode**: enforce `ES256` + KMS/HSM only; disallow Ed25519.
* **Dual attestation**: optional immediate push to **Attestor** (sync mode) with timeout budget, returning Rekor UUID inline.

342
docs/ARCHITECTURE_UI.md Normal file
View File

@@ -0,0 +1,342 @@
# component_architecture_web_ui.md — **StellaOps Web UI** (2025Q4)
> **Scope.** Implementationready architecture for the **Angular SPA** that operators and developers use to drive StellaOps. This document defines UX surfaces, module boundaries, data flows, auth, RBAC, realtime updates, performance targets, i18n/a11y, security headers, testing and deployment. The UI is a *consumer* of backend APIs (Scanner, Policy, Excititor, Concelier, Attestor, Authority) and never performs scanning, merging, or signing on its own.
---
## 0) Mission & nongoals
**Mission.** Provide a **fast, explainable** console for:
* Scans (status, SBOMs, diffs, EntryTrace, attestation).
* Policy management (rules, exemptions, VEX consumption view).
* Vulnerability intel (Concelier status), VEX consensus exploration (Excititor).
* Runtime posture (Zastava observer + admission).
* Admin operations (tenants, tokens, quotas, licensing posture).
**Nongoals.** No clientside crypto signing; no Docker/CRI access; no direct registry access beyond fetching static assets or OCI referrer summaries exposed by backend.
---
## 1) Technology baseline
* **Framework**: Angular 17+ (Standalone APIs / Signals), TypeScript 5.
* **Styling**: Tailwind CSS + headless component patterns; CSS variables for theming.
* **Charts**: Lightweight SVG (uPlot or Apache ECharts via ondemand import).
* **State**: Angular **Signals** + `@ngrx/signals` store for crosspage slices.
* **Transport**: `fetch` + RxJS interop; **SSE** (EventSource) for progress streams.
* **Build**: Angular CLI + Vite builder.
* **Testing**: Jest + Testing Library, Playwright for e2e.
* **Packaging**: Containerized NGINX (immutable assets, ETag + content hashing).
---
## 2) Highlevel module map
```
/app
├─ core/ # bootstrap, config, auth, http, error boundary, i18n
├─ shared/ # UI kit (tables, code-viewers, badges), pipes
├─ dashboard/ # live tiles, fleet KPIs, feed/vex age, queue depth
├─ scans/ # scan list, detail, SBOM viewer, diff-by-layer, EntryTrace
├─ runtime/ # Zastava posture, drift events, admission decisions
├─ policy/ # rules editor (YAML/Rego), exemptions, previews
├─ vex/ # VEX explorer (claims, consensus, conflicts)
├─ concelier/ # source health, export cursors, rebuild/export triggers
├─ attest/ # attestation proofs, verification bundles, Rekor links
├─ admin/ # tenants, roles, clients, quotas, licensing posture
└─ plugins/ # route plug-ins (lazy remote modules, governed)
```
Each feature folder builds as a **standalone route** (lazy loaded). All HTTP shapes live in `core/api/` clients with shared DTOs.
---
## 3) Navigation & key views
### 3.1 Dashboard
* **Tiles**: “New criticals (24h)”, “VEX suppressions applied”, “Attested SBOMs (7d)”, “Feed age per provider”, “Scanner queue depth”, “Admission events”.
* **Trends**: sparkline for vulns/day, pass/fail rates, attestation throughput.
### 3.2 Scans
* **Scan list** with status, image digest, repo, time, artifacts, attestation badge.
* **Scan detail**:
* **SBOM viewer**: Inventory/Usage toggle; component table (virtualized), filters by package type, severity, source.
* **Diff by layer**: A→B change grid (added/removed/upgraded), grouped by introducing/removing layer; tooltips show provenance and links to layer SBOM fragment.
* **EntryTrace**: shell chain with file:line breadcrumbs; jumpto source viewer (readonly, hexdump fallback).
* **Attestation**: Rekor UUID, index, inclusion proof; **Verify** button calls Attestor `/verify`.
* **Export**: download buttons (CycloneDX JSON, Protobuf, SPDX JSON); size shown; SHA256 inline.
### 3.3 Runtime (Zastava)
* **Observer timeline**: container start/stop, drift, policy violations; faceted by namespace/owner.
* **Live process view**: top N processes, loaded libs summary vs Usage SBOM.
* **Admission decisions**: pernamespace rules, allow/deny events, cache TTL, reasons.
### 3.4 Policy
* **Policy bundles**: active vs staged; diff viewer with change summary.
* **Editors**:
* YAML rules (ignore lists, thresholds, vendor precedence overrides).
* Rego blocks (advanced gates) with **WASM** preview evaluator (clientside sandbox) for “preview” against sample SBOMs.
* **VEX inclusion controls**: weight sliders (visualization only), provider allow/deny toggles.
* **Preview**: select SBOM (or image digest) → show verdict under staged policy.
### 3.5 Excititor
* **Claims explorer**: search by vulnId/productKey/provider; show raw claim (status, justification, evidence).
* **Consensus view**: rollup per (vuln, product) with accepted/rejected sources, weights, timestamps.
* **Conflicts**: grid of top conflicts; filters for justification gates failed.
### 3.6 Concelier
* **Sources** table: staleness, last run, errors.
* **Advisory search**: by CVE/alias; show normalized affected ranges.
* **Exports**: trigger full/delta JSON/Trivy DB; show manifest digests and Rekor link if attested.
### 3.7 Attest
* **Proofs list**: last 7 days Rekor entries; filter by kind (sbom/report/vex).
* **Verification**: paste UUID or upload bundle → verify; result with explanations (chain, Merkle path).
### 3.8 Admin
* **Tenants/Installations**: view/edit, isolation hints.
* **Clients & roles**: Authority clients, role→scope mapping, rotation hints.
* **Quotas**: per license plan, counters, throttle events.
* **Licensing posture**: last PoE introspection snapshot (redacted), release window.
---
## 4) Auth, sessions & RBAC
### 4.1 OIDC flow
* **Authorization Code + PKCE** to **Authority**.
* **ID Token** for UX identity; **Access Token** (OpTok) for APIs (25min TTL).
* **DPoP (browser)**: generate ephemeral **WebCrypto** keypair; store public JWK in memory, private key in **IndexedDB** (nonexportable if platform allows). Access token includes `cnf.jkt`; each API call adds `DPoP` proof; handle nonce challenges automatically.
* **Refresh**: optional DPoPbound refresh tokens with rotation; otherwise silent renew.
### 4.2 RBAC
* Roles (`ui.read`, `ui.admin`, plus service roles) are embedded in ID token or fetched via `/me` endpoint.
* **Route guards** enforce access; **feature flags** hide admin pages for nonadmins.
### 4.3 Session storage
* Access tokens & refresh metadata in memory; persist **only** minimal session (subject, expiries) in `sessionStorage`. Never persist raw JWTs to `localStorage`. Use **SameSite=Lax** cookies for antiCSRF if cookies are required (prefer pure bearer headers + DPoP).
---
## 5) HTTP layer & API clients
* **`core/http/api-client.ts`** centralizes:
* Base URLs (Scanner, Excititor, Concelier, Attestor).
* **Retry** policies on idempotent GETs (backoff + jitter).
* **Problem+JSON** parser → uniform error toasts with correlation ID.
* **SSE** helper (EventSource) with autoreconnect & backpressure.
* **DPoP** injector & nonce handling.
* Typed API clients (DTOs in `core/api/models.ts`):
* `ScannerApi`, `PolicyApi`, `ExcititorApi`, `ConcelierApi`, `AttestorApi`, `AuthorityApi`.
**DTO examples (abbrev):**
```ts
export type ImageDigest = `sha256:${string}`;
export interface ScanSummary {
imageDigest: ImageDigest; createdAt: string;
artifacts: { view: 'inventory'|'usage'; format: 'cdx-json'|'cdx-pb'|'spdx-json'; sha256: string; size: number }[];
status: 'queued'|'running'|'completed'|'error';
rekor?: { uuid: string; index?: number; url?: string };
}
export interface DiffEntry {
key: string; change: 'added'|'removed'|'upgraded'|'downgraded';
fromVersion?: string; toVersion?: string; layer: string; usedByEntrypoint?: boolean;
}
export interface VexConsensus {
vulnId: string; productKey: string; rollupStatus: 'affected'|'not_affected'|'fixed'|'under_investigation';
sources: { providerId: string; status: string; weight: number; accepted: boolean; reason: string }[];
}
```
---
## 6) State, caching & realtime
* **Perpage stores** (Signals) for list filters, pagination, and selected entities.
* **Normalized caches** keyed by `(imageDigest, view, format)`; artifacts are downloaded via presigned URLs from Scanner and streamed; SHA256 verified clientside before exposing “verified” badge.
* **SSE channels**:
* `/scans/{id}/events` → progress log.
* `/runtime/events/stream` (optional) → live drift/admission feed (ratelimited).
* **Cache invalidation** on job completion or explicit “refresh”.
---
## 7) SBOM viewing & diff UX
* **Huge tables** rendered with **virtual scrolling** (CDK Virtual Scroll); sort/filter performed clientside for ≤ 20k rows; beyond that, serverside queries via BOMIndex endpoints.
* **Component row** shows purl, version, origin (OS pkg / metadata / linker / attested), licenses, and **used** badge (Usage view).
* **Diff**: compact heatmap per layer; clicking opens a rightpane with evidence: introducing paths, file hashes, VEX notes (from Excititor consensus) and links to advisories (Concelier).
---
## 8) Policy editor & VEX integration
* **YAML editor**: Monacobased with schema hints; previews show which rules matched.
* **Rego editor**: Monaco with WASMOPA sandbox for readonly evaluation on sample SBOMs.
* **VEX toggles**: perprovider enable/disable; “explain” drawer shows why a claim was accepted/rejected (justification gate, signature state, weight).
* **Staged → Active** promotion is a twostep flow with confirmation and automatic policy digest computation.
---
## 9) Accessibility, i18n & theming
* **A11y**: WCAG 2.2 AA; keyboard navigation, focus management, ARIA roles; colorcontrast tokens verified by unit tests.
* **I18n**: Angular i18n + runtime translation loader (`/locales/{lang}.json`); dates/numbers localized via `Intl`.
* **Languages**: English default; Bulgarian, German, Japanese as initial additions.
* **Theming**: dark/light via CSS variables; persisted in `prefers-color-scheme` aware store.
---
## 10) Performance budgets
* **TTI** ≤ 1.5s on 4G/slow CPU (first visit), ≤ 0.6s repeat (HTTP/2, cached).
* **JS** initial < 300KB gz (lazy routes).
* **SBOM list**: render 10k rows in < 70ms with virtualization; filter in < 150ms.
* **Diff view**: compute clientside grouping for 5k changes in < 120ms.
Techniques: routelevel code splitting; `ChangeDetectionStrategy.OnPush`; Signals; server compression (zstd/gzip), immutable assets with long maxage (cache busting via hashes).
---
## 11) Security headers & CSP
* **CSP**: `default-src 'self'; connect-src 'self' https://*.internal; img-src 'self' data:; script-src 'self'; style-src 'self' 'unsafe-inline'; frame-ancestors 'none';`
* **HSTS** enabled at gateway.
* **Referrer Policy**: `no-referrer`.
* **XFrameOptions**: `DENY`.
* **COOP/COEP** to enable faster crossorigin isolation (for WASM OPA).
* **Subresource Integrity (SRI)** for thirdparty fonts (minimize thirdparty).
---
## 12) Error handling & UX hygiene
* **Global error boundary** surfaces Problem+JSON `title/detail/instance` with correlation ID.
* **Retry toast** for 429 (quota throttles) with backoff timer.
* **Auth expiry**: preemptive refresh; unobtrusive banner when < 60s TTL; relogin modal if refresh fails.
* **Network down**: offline banner with queued actions (idempotent resubmits).
* **File verify**: show SHA256 mismatch warnings if artifact altered in transit.
---
## 13) Observability
* **Frontend telemetry** (OpenTelemetry Web): route timings, API latency by service, error counts; sampled to 15% and shipped to backend OTLP endpoint.
* **User actions** logged anonymously (no PII): policy promote”, scan export”, attest verify”.
* **Metrics dash** in admin shows SLOs and recent frontend errors.
---
## 14) Testing strategy
* **Unit**: pure component logic via Jest + Testing Library (no TestBed when possible).
* **Component harness** for table, code viewer, diff heatmap.
* **Contract tests**: OpenAPI schemas pulled at build time; DTOs validated; breaking changes fail CI.
* **e2e**: Playwright scenarios (login, scan detail, diff, policy edit, admit deny).
* **A11y**: axe-core CI checks; colorcontrast lints.
* **i18n**: key coverage tests (no missing translations in supported locales).
---
## 15) Deployment & ops
* **Container**: `stellaops/web-ui:<ver>-<rev>`; NGINX with `gzip_static` + brotli; immutable assets under `/static/<hash>/…`.
* **Config**: `/config.json` served by gateway (injected at runtime): API base URLs, authority issuer, telemetry sampling.
* **Version banner**: footer shows UI & backend versions; warns on major mismatches.
* **CDN** (optional): cache static bundle; APIs stay behind internal gateway.
* **Feature flags**: environment gates (staged policies, eBPF runtime) readable from config.
---
## 16) Plugin system (route plugins)
* **Manifest**: Backend provides a signed plugin manifest with remote module URLs and **cosign signature** per JS bundle.
* **Loader**: dynamic import with **SRI** and signature verification (WebCrypto).
* **Sandbox**: plugins are routed modules receiving a limited **UI SDK** (navigation, theme, API gateway). No direct token access; API calls proxied through the UI SDK which enforces RBAC.
* **Examples**: custom reports, vendor dashboards, regulated TLS config UIs.
---
## 17) Wire sequences (representative)
**A) View scan progress**
```mermaid
sequenceDiagram
autonumber
participant UI
participant Auth as Authority
participant SW as Scanner.WebService
UI->>Auth: /authorize (PKCE)
Auth-->>UI: code → token (DPoP-bound)
UI->>SW: GET /scans/{id} (Authorization+DPoP)
SW-->>UI: { status: running }
UI->>SW: (SSE) GET /scans/{id}/events
SW-->>UI: progress events …
SW-->>UI: terminal event { status: completed, artifacts[] }
```
**B) Verify attestation**
```mermaid
sequenceDiagram
autonumber
participant UI
participant AT as Attestor
UI->>AT: POST /rekor/verify { uuid }
AT-->>UI: { ok:true, index, logURL }
```
**C) Promote policy & preview**
```mermaid
sequenceDiagram
autonumber
participant UI
participant BE as Scanner.WebService (Policy endpoint)
UI->>BE: POST /policy/stage { yaml, rego }
BE-->>UI: { policyRevision, diagnostics }
UI->>BE: POST /policy/preview { imageDigest, policyRevision }
BE-->>UI: { verdict: pass|fail, reasons[] }
UI->>BE: POST /policy/promote { policyRevision }
BE-->>UI: { ok:true }
```
---
## 18) Security hard lines
* Never store JWTs in `localStorage`.
* Enforce DPoP for API calls; if DPoP unsupported for a service, require **SameSite=Lax** cookies with CSRF token header.
* Block mixedcontent; only HTTPS origins allowed.
* Validate and render only **escaped** user content; code viewer uses safe highlighter.
* Downloaded artifacts are treated as **opaque binaries**; no HTML rendering.
---
## 19) Roadmap
* **PWA** offline shell (readonly) for dashboards and cached scan details.
* **SBOM graph** visualization (forcedirected) for small components sets.
* **Runtime session replay** (privacysafe) to debug operator workflows (optin).
* **Assistive wizards** for policy creation with guided templates.

View File

@@ -0,0 +1,451 @@
# component_architecture_zastava.md — **StellaOps Zastava** (2025Q4)
> **Scope.** Implementationready architecture for **Zastava**: the **runtime inspector/enforcer** that watches real workloads, detects drift from the scanned baseline, verifies image/SBOM/attestation posture, and (optionally) **admits/blocks** deployments. Includes Kubernetes & plainDocker topologies, data contracts, APIs, security posture, performance targets, test matrices, and failure modes.
---
## 0) Mission & boundaries
**Mission.** Give operators **groundtruth** from running environments and a **fast guardrail** before workloads land:
* **Observer:** inventory containers, entrypoints actually executed, and DSOs actually loaded; verify **image signature**, **SBOM referrers**, and **attestation** presence; detect **drift** (unexpected processes/paths) and **policy violations**; publish **runtime events** to Scanner.WebService.
* **Admission (optional):** Kubernetes ValidatingAdmissionWebhook that enforces minimal posture (signed images, SBOM availability, known base images, policy PASS) **preflight**.
**Boundaries.**
* Zastava **does not** compute SBOMs and does not sign; it **consumes** Scanner/WebService outputs and **enforces** backend policy verdicts.
* Zastava can **request** a delta scan when the baseline is missing/stale, but scanning is done by **Scanner.Worker**.
* On nonK8s Docker hosts, Zastava runs as a host service with **observeronly** features.
---
## 1) Topology & processes
### 1.1 Components (Kubernetes)
```
stellaops/zastava-observer # DaemonSet on every node (read-only host mounts)
stellaops/zastava-webhook # ValidatingAdmissionWebhook (Deployment, 2+ replicas)
```
### 1.2 Components (Docker/VM)
```
stellaops/zastava-agent # System service; watch Docker events; observer only
```
### 1.3 Dependencies
* **Authority** (OIDC): short OpToks (DPoP/mTLS) for API calls to Scanner.WebService.
* **Scanner.WebService**: `/runtime/events` ingestion; `/policy/runtime` fetch.
* **OCI Registry** (optional): for direct referrers/sig checks if not delegated to backend.
* **Container runtime**: containerd/CRIO/Docker (read interfaces only).
* **Kubernetes API** (watch Pods in cluster; validating webhook).
* **Host mounts** (K8s DaemonSet): `/proc`, `/var/lib/containerd` (or CRIO), `/run/containerd/containerd.sock` (optional, readonly).
---
## 2) Data contracts
### 2.1 Runtime event (observer → Scanner.WebService)
```json
{
"eventId": "9f6a…",
"when": "2025-10-17T12:34:56Z",
"kind": "CONTAINER_START|CONTAINER_STOP|DRIFT|POLICY_VIOLATION|ATTESTATION_STATUS",
"tenant": "tenant-01",
"node": "ip-10-0-1-23",
"runtime": { "engine": "containerd", "version": "1.7.19" },
"workload": {
"platform": "kubernetes",
"namespace": "payments",
"pod": "api-7c9fbbd8b7-ktd84",
"container": "api",
"containerId": "containerd://...",
"imageRef": "ghcr.io/acme/api@sha256:abcd…",
"owner": { "kind": "Deployment", "name": "api" }
},
"process": {
"pid": 12345,
"entrypoint": ["/entrypoint.sh", "--serve"],
"entryTrace": [
{"file":"/entrypoint.sh","line":3,"op":"exec","target":"/usr/bin/python3"},
{"file":"<argv>","op":"python","target":"/opt/app/server.py"}
]
},
"loadedLibs": [
{ "path": "/lib/x86_64-linux-gnu/libssl.so.3", "inode": 123456, "sha256": "…"},
{ "path": "/usr/lib/x86_64-linux-gnu/libcrypto.so.3", "inode": 123457, "sha256": "…"}
],
"posture": {
"imageSigned": true,
"sbomReferrer": "present|missing",
"attestation": { "uuid": "rekor-uuid", "verified": true }
},
"delta": {
"baselineImageDigest": "sha256:abcd…",
"changedFiles": ["/opt/app/server.py"], // optional quick signal
"newBinaries": [{ "path":"/usr/local/bin/helper","sha256":"…" }]
},
"evidence": [
{"signal":"procfs.maps","value":"/lib/.../libssl.so.3@0x7f..."},
{"signal":"cri.task.inspect","value":"pid=12345"},
{"signal":"registry.referrers","value":"sbom: application/vnd.cyclonedx+json"}
]
}
```
### 2.2 Admission decision (webhook → API server)
```json
{
"admissionId": "…",
"namespace": "payments",
"podSpecDigest": "sha256:…",
"images": [
{
"name": "ghcr.io/acme/api:1.2.3",
"resolved": "ghcr.io/acme/api@sha256:abcd…",
"signed": true,
"hasSbomReferrers": true,
"policyVerdict": "pass|warn|fail",
"reasons": ["unsigned base image", "missing SBOM"]
}
],
"decision": "Allow|Deny",
"ttlSeconds": 300
}
```
---
## 3) Observer — node agent (DaemonSet)
### 3.1 Responsibilities
* **Watch** container lifecycle (start/stop) via CRI (`/run/containerd/containerd.sock` gRPC readonly) or `/var/log/containers/*.log` tail fallback.
* **Resolve** container → image digest, mount point rootfs.
* **Trace entrypoint**: attach **shortlived** nsenter/exec to PID 1 in container, parse shell for `exec` chain (bounded depth), record **terminal program**.
* **Sample loaded libs**: read `/proc/<pid>/maps` and `exe` symlink to collect **actually loaded** DSOs; compute **sha256** for each mapped file (bounded count/size).
* **Posture check** (cheap):
* Image signature presence (if cosign policies are local; else ask backend).
* SBOM **referrers** presence (HEAD to registry, optional).
* Rekor UUID known (query Scanner.WebService by image digest).
* **Publish runtime events** to Scanner.WebService `/runtime/events` (batch & compress).
* **Request delta scan** if: no SBOM in catalog OR base differs from known baseline.
### 3.2 Privileges & mounts (K8s)
* **SecurityContext:** `runAsUser: 0`, `readOnlyRootFilesystem: true`, `allowPrivilegeEscalation: false`.
* **Capabilities:** `CAP_SYS_PTRACE` (optional if using nsenter trace), `CAP_DAC_READ_SEARCH`.
* **Host mounts (readonly):**
* `/proc` (host) → `/host/proc`
* `/run/containerd/containerd.sock` (or CRIO socket)
* `/var/lib/containerd/io.containerd.runtime.v2.task` (rootfs paths & pids)
* **Networking:** clusterinternal egress to Scanner.WebService only.
* **Rate limits:** hard caps for bytes hashed and file count per container to avoid noisy tenants.
### 3.3 Event batching
* Buffer NDJSON; flush by **N events** or **2s**.
* Backpressure: local disk ring buffer (50MB default) if Scanner is temporarily unavailable; drop oldest after cap with **metrics** and **warning** event.
---
## 4) Admission Webhook (Kubernetes)
### 4.1 Gate criteria
Configurable policy (fetched from backend and cached):
* **Image signature**: must be cosignverifiable to configured key(s) or keyless identities.
* **SBOM availability**: at least one **CycloneDX** referrer or **Scanner.WebService** catalog entry.
* **Scanner policy verdict**: backend `PASS` required for namespaces/labels matching rules; allow `WARN` if configured.
* **Registry allowlists/denylists**.
* **Tag bans** (e.g., `:latest`).
* **Base image allowlists** (by digest).
### 4.2 Flow
```mermaid
sequenceDiagram
autonumber
participant K8s as API Server
participant WH as Zastava Webhook
participant SW as Scanner.WebService
K8s->>WH: AdmissionReview(Pod)
WH->>WH: Resolve images to digests (remote HEAD/pull if needed)
WH->>SW: POST /policy/runtime { digests, namespace, labels }
SW-->>WH: { per-image: {signed, hasSbom, verdict, reasons}, ttl }
alt All pass
WH-->>K8s: AdmissionResponse(Allow, ttl)
else Any fail (enforce=true)
WH-->>K8s: AdmissionResponse(Deny, message)
end
```
**Caching:** Perdigest result cached `ttlSeconds` (default 300 s). **Failopen** or **failclosed** is configurable per namespace.
### 4.3 TLS & HA
* Webhook has its own **serving cert** signed by cluster CA (or custom cert + CA bundle on configuration).
* Deployment ≥ 2 replicas; **leaderless**; stateless.
---
## 5) Backend integration (Scanner.WebService)
### 5.1 Ingestion endpoint
`POST /api/v1/scanner/runtime/events` *(OpTok + DPoP/mTLS)*
* Validates event schema; enforces rate caps by tenant/node; persists to **Mongo** (`runtime.events` capped collection or regular with TTL).
* Performs **correlation**:
* Attach nearest **image SBOM** (inventory/usage) and **BOMIndex** if known.
* If unknown/missing, schedule **delta scan** and return `202 Accepted`.
* Emits **derived signals** (usedByEntrypoint per component based on `/proc/<pid>/maps`).
### 5.2 Policy decision API (for webhook)
`POST /api/v1/scanner/policy/runtime`
Request:
```json
{
"namespace": "payments",
"labels": { "app": "api", "env": "prod" },
"images": ["ghcr.io/acme/api@sha256:...", "ghcr.io/acme/nginx@sha256:..."]
}
```
Response:
```json
{
"ttlSeconds": 300,
"results": {
"ghcr.io/acme/api@sha256:...": {
"signed": true,
"hasSbom": true,
"policyVerdict": "pass",
"reasons": [],
"rekor": { "uuid": "..." }
},
"ghcr.io/acme/nginx@sha256:...": {
"signed": false,
"hasSbom": false,
"policyVerdict": "fail",
"reasons": ["unsigned", "missing SBOM"]
}
}
}
```
---
## 6) Configuration (YAML)
```yaml
zastava:
mode:
observer: true
webhook: true
authority:
issuer: "https://authority.internal"
aud: ["scanner","zastava"] # tokens for backend and self-id
backend:
url: "https://scanner-web.internal"
connectTimeoutMs: 500
requestTimeoutMs: 1500
retry: { attempts: 3, backoffMs: 200 }
runtime:
engine: "auto" # containerd|cri-o|docker|auto
procfs: "/host/proc"
collect:
entryTrace: true
loadedLibs: true
maxLibs: 256
maxHashBytesPerContainer: 64_000_000
maxDepth: 48
admission:
enforce: true
failOpenNamespaces: ["dev", "test"]
verify:
imageSignature: true
sbomReferrer: true
scannerPolicyPass: true
cacheTtlSeconds: 300
resolveTags: true # do remote digest resolution for tag-only images
limits:
eventsPerSecond: 50
burst: 200
perNodeQueue: 10_000
security:
mounts:
containerdSock: "/run/containerd/containerd.sock:ro"
proc: "/proc:/host/proc:ro"
runtimeState: "/var/lib/containerd:ro"
```
---
## 7) Security posture
* **AuthN/Z**: Authority OpToks (DPoP preferred) to backend; webhook does **not** require client auth from API server (K8s handles).
* **Least privileges**: readonly host mounts; optional `CAP_SYS_PTRACE`; **no** host networking; **no** write mounts.
* **Isolation**: never exec untrusted code; nsenter only to **read** `/proc/<pid>`.
* **Data minimization**: do not exfiltrate env vars or command arguments unless policy explicitly enables diagnostic mode.
* **Rate limiting**: pernode caps; pertenant caps at backend.
* **Hard caps**: bytes hashed, files inspected, depth of shell parsing.
---
## 8) Metrics, logs, tracing
**Observer**
* `zastava.events_emitted_total{kind}`
* `zastava.proc_maps_samples_total{result}`
* `zastava.entrytrace_depth{p99}`
* `zastava.hash_bytes_total`
* `zastava.buffer_drops_total`
**Webhook**
* `zastava.admission_requests_total{decision}`
* `zastava.admission_latency_seconds`
* `zastava.cache_hits_total`
* `zastava.backend_failures_total`
**Logs** (structured): node, pod, image digest, decision, reasons.
**Tracing**: spans for observe→batch→post; webhook request→resolve→respond.
---
## 9) Performance & scale targets
* **Observer**: ≤ **30ms** to sample `/proc/<pid>/maps` and compute quick hashes for ≤ 64 files; ≤ **200ms** for full library set (256 libs).
* **Webhook**: P95 ≤ **8ms** with warm cache; ≤ **50ms** with one backend roundtrip.
* **Throughput**: 1k admission requests/min/replica; 5k runtime events/min/node with batching.
---
## 10) Drift detection model
**Signals**
* **Process drift**: terminal program differs from **EntryTrace** baseline.
* **Library drift**: loaded DSOs not present in **Usage** SBOM view.
* **Filesystem drift**: new executable files under `/usr/local/bin`, `/opt`, `/app` with **mtime** after image creation.
* **Network drift** (optional): listening sockets on unexpected ports (from policy).
**Action**
* Emit `DRIFT` event with evidence; backend can **autoqueue** a delta scan; policy may **escalate** to alert/block (Admission cannot block alreadyrunning pods; rely on K8s policies/PodSecurity or operator action).
---
## 11) Test matrix
* **Engines**: containerd, CRIO, Docker; ensure PID resolution and rootfs mapping.
* **EntryTrace**: bash features (case, if, runparts, `.`/`source`), language launchers (python/node/java).
* **Procfs**: multiple arches, musl/glibc images; static binaries (maps minimal).
* **Admission**: unsigned images, missing SBOM referrers, tagonly images, digest resolution, backend latency, cache TTL.
* **Perf/soak**: 500 Pods/node churn; webhook under HPA growth.
* **Security**: attempt privilege escalation disabled, readonly mounts enforced, ratelimit abuse.
* **Failure injection**: backend down (observer buffers, webhook failopen/closed), registry throttling, containerd socket unavailable.
---
## 12) Failure modes & responses
| Condition | Observer behavior | Webhook behavior |
| ------------------------------- | ---------------------------------------------- | ------------------------------------------------------ |
| Backend unreachable | Buffer to disk; drop after cap; emit metric | **Failopen/closed** per namespace config |
| PID vanished midsample | Retry once; emit partial evidence | N/A |
| CRI socket missing | Fallback to K8s events only (reduced fidelity) | N/A |
| Registry digest resolve blocked | Defer to backend; mark `resolve=unknown` | Deny or allow per `resolveTags` & `failOpenNamespaces` |
| Excessive events | Apply local rate limit, coalesce | N/A |
---
## 13) Deployment notes (K8s)
**DaemonSet (snippet):**
```yaml
apiVersion: apps/v1
kind: DaemonSet
metadata: { name: zastava-observer, namespace: stellaops }
spec:
template:
spec:
serviceAccountName: zastava
hostPID: true
containers:
- name: observer
image: stellaops/zastava-observer:2.3
securityContext:
runAsUser: 0
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
capabilities: { add: ["SYS_PTRACE","DAC_READ_SEARCH"] }
volumeMounts:
- { name: proc, mountPath: /host/proc, readOnly: true }
- { name: containerd-sock, mountPath: /run/containerd/containerd.sock, readOnly: true }
- { name: containerd-state, mountPath: /var/lib/containerd, readOnly: true }
volumes:
- { name: proc, hostPath: { path: /proc } }
- { name: containerd-sock, hostPath: { path: /run/containerd/containerd.sock } }
- { name: containerd-state, hostPath: { path: /var/lib/containerd } }
```
**Webhook (snippet):**
```yaml
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
webhooks:
- name: gate.zastava.stella-ops.org
admissionReviewVersions: ["v1"]
sideEffects: None
failurePolicy: Ignore # or Fail
rules:
- operations: ["CREATE","UPDATE"]
apiGroups: [""]
apiVersions: ["v1"]
resources: ["pods"]
clientConfig:
service:
namespace: stellaops
name: zastava-webhook
path: /admit
caBundle: <base64 CA>
```
---
## 14) Implementation notes
* **Language**: Rust (observer) for lowlatency `/proc` parsing; Go/.NET viable too. Webhook can be .NET 10 for parity with backend.
* **CRI drivers**: pluggable (`containerd`, `cri-o`, `docker`). Prefer CRI over parsing logs.
* **Shell parser**: reuse Scanner.EntryTrace grammar for consistent results (compile to WASM if observer is Rust/Go).
* **Hashing**: `BLAKE3` for speed locally, then convert to `sha256` (or compute `sha256` directly when budget allows).
* **Resilience**: never block container start; observer is **passive**; only webhook decides allow/deny.
---
## 15) Roadmap
* **eBPF** option for syscall/library load tracing (kernellevel, optin).
* **Windows containers** support (ETW providers, loaded modules).
* **Network posture** checks: listening ports vs policy.
* **Live **usedbyentrypoint** synthesis**: send compact bitset diff to backend to tighten Usage view.
* **Admission dryrun** dashboards (simulate block lists before enforcing).

View File

@@ -0,0 +1,83 @@
## Status
This document tracks the future-looking risk scoring model for Excititor. The calculation below is not active yet; Sprint 7 work will add the required schema fields, policy controls, and services. Until that ships, Excititor emits consensus statuses without numeric scores.
## Scoring model (target state)
**S = Gate(VEX_status) × W_trust(source) × [Severity_base × (1 + α·KEV + β·EPSS)]**
* **Gate(VEX_status)**: `affected`/`under_investigation` → 1, `not_affected`/`fixed` → 0. A trusted “not affected” or “fixed” still zeroes the score.
* **W_trust(source)**: normalized policy weight (baseline 01). Policies may opt into >1 boosts for signed vendor feeds once Phase 1 closes.
* **Severity_base**: canonical numeric severity from Concelier (CVSS or org-defined scale).
* **KEV flag**: 0/1 boost when CISA Known Exploited Vulnerabilities applies.
* **EPSS**: probability [0,1]; bounded multiplier.
* **α, β**: configurable coefficients (default α=0.25, β=0.5) stored in policy.
Safeguards: freeze boosts when product identity is unknown, clamp outputs ≥0, and log every factor in the audit trail.
## Implementation roadmap
| Phase | Scope | Artifacts |
| --- | --- | --- |
| **Phase 1 Schema foundations** | Extend Excititor consensus/claims and Concelier canonical advisories with severity, KEV, EPSS, and expose α/β + weight ceilings in policy. | Sprint 7 tasks `EXCITITOR-CORE-02-001`, `EXCITITOR-POLICY-02-001`, `EXCITITOR-STORAGE-02-001`, `FEEDCORE-ENGINE-07-001`. |
| **Phase 2 Deterministic score engine** | Implement a scoring component that executes alongside consensus and persists score envelopes with hashes. | Planned task `EXCITITOR-CORE-02-002` (backlog). |
| **Phase 3 Surfacing & enforcement** | Expose scores via WebService/CLI, integrate with Concelier noise priors, and enforce policy-based suppressions. | To be scheduled after Phase 2. |
## Data model (after Phase 1)
```json
{
"vulnerabilityId": "CVE-2025-12345",
"product": "pkg:name@version",
"consensus": {
"status": "affected",
"policyRevisionId": "rev-12",
"policyDigest": "0D9AEC…"
},
"signals": {
"severity": {"scheme": "CVSS:3.1", "score": 7.5},
"kev": true,
"epss": 0.40
},
"policy": {
"weight": 1.15,
"alpha": 0.25,
"beta": 0.5
},
"score": {
"value": 10.8,
"generatedAt": "2025-11-05T14:12:30Z",
"audit": [
"gate:affected",
"weight:1.15",
"severity:7.5",
"kev:1",
"epss:0.40"
]
}
}
```
## Operational guidance
* **Inputs**: Concelier delivers severity/KEV/EPSS via the advisory event log; Excititor connectors load VEX statements. Policy owns trust tiers and coefficients.
* **Processing**: the scoring engine (Phase 2) runs next to consensus, storing results with deterministic hashes so exports and attestations can reference them.
* **Consumption**: WebService/CLI will return consensus plus score; scanners may suppress findings only when policy-authorized VEX gating and signed score envelopes agree.
## Pseudocode (Phase 2 preview)
```python
def risk_score(gate, weight, severity, kev, epss, alpha, beta, freeze_boosts=False):
if gate == 0:
return 0
if freeze_boosts:
kev, epss = 0, 0
boost = 1 + alpha * kev + beta * epss
return max(0, weight * severity * boost)
```
## FAQ
* **Can operators opt out?** Set α=β=0 or keep weights ≤1.0 via policy.
* **What about missing signals?** Treat them as zero and log the omission.
* **When will this ship?** Phase 1 is planned for Sprint 7; later phases depend on connector coverage and attestation delivery.

View File

@@ -31,17 +31,26 @@ Everything here is opensource and versioned— when you check out a git ta
- **03[Vision & Roadmap](03_VISION.md)**
- **04[Feature Matrix](04_FEATURE_MATRIX.md)**
### Reference & concepts
- **05[System Requirements Specification](05_SYSTEM_REQUIREMENTS_SPEC.md)**
- **07[HighLevel Architecture](40_ARCHITECTURE_OVERVIEW.md)**
- **08Module Specifications**
- [README](08_MODULE_SPECIFICATIONS/README.md)
- [`backend_api.md`](08_MODULE_SPECIFICATIONS/backend_api.md)
- [`zastava_scanner.md`](08_MODULE_SPECIFICATIONS/zastava_scanner.md)
- [`registry_scanner.md`](08_MODULE_SPECIFICATIONS/registry_scanner.md)
- [`nightly_scheduler.md`](08_MODULE_SPECIFICATIONS/nightly_scheduler.md)
### Reference & concepts
- **05[System Requirements Specification](05_SYSTEM_REQUIREMENTS_SPEC.md)**
- **07[HighLevel Architecture](07_HIGH_LEVEL_ARCHITECTURE.md)**
- **08Module Architecture Dossiers**
- [Scanner](ARCHITECTURE_SCANNER.md)
- [Concelier](ARCHITECTURE_CONCELIER.md)
- [Excititor](ARCHITECTURE_EXCITITOR.md)
- [Signer](ARCHITECTURE_SIGNER.md)
- [Attestor](ARCHITECTURE_ATTESTOR.md)
- [Authority](ARCHITECTURE_AUTHORITY.md)
- [CLI](ARCHITECTURE_CLI.md)
- [WebUI](ARCHITECTURE_UI.md)
- [Zastava Runtime](ARCHITECTURE_ZASTAVA.md)
- [Release & Operations](ARCHITECTURE_DEVOPS.md)
- **09[API&CLI Reference](09_API_CLI_REFERENCE.md)**
- **10[Plugin SDK Guide](10_PLUGIN_SDK_GUIDE.md)**
- **10[Concelier CLI Quickstart](10_CONCELIER_CLI_QUICKSTART.md)**
- **30[Excititor Connector Packaging Guide](dev/30_EXCITITOR_CONNECTOR_GUIDE.md)**
- **30Developer Templates**
- [Excititor Connector Skeleton](dev/templates/excititor-connector/)
- **11[Authority Service](11_AUTHORITY.md)**
- **11[Data Schemas](11_DATA_SCHEMAS.md)**
- **12[Performance Workbook](12_PERFORMANCE_WORKBOOK.md)**
@@ -57,12 +66,16 @@ Everything here is opensource and versioned— when you check out a git ta
- **21[Install Guide](21_INSTALL_GUIDE.md)**
- **22[CI/CD Recipes Library](ci/20_CI_RECIPES.md)**
- **23[FAQ](23_FAQ_MATRIX.md)**
- **24[Offline Update Kit Admin Guide](24_OUK_ADMIN_GUIDE.md)**
- **24[Offline Update Kit Admin Guide](24_OFFLINE_KIT.md)**
- **25[Concelier Apple Connector Operations](ops/concelier-apple-operations.md)**
- **26[Authority Key Rotation Playbook](ops/authority-key-rotation.md)**
- **25[Feedser Apple Connector Operations](ops/feedser-apple-operations.md)**
### Legal & licence
- **29[Legal & Quota FAQ](29_LEGAL_FAQ_QUOTA.md)**
- **27[Concelier CCCS Connector Operations](ops/concelier-cccs-operations.md)**
- **28[Concelier CISA ICS Connector Operations](ops/concelier-icscisa-operations.md)**
- **29[Concelier CERT-Bund Connector Operations](ops/concelier-certbund-operations.md)**
- **30[Concelier MSRC Connector AAD Onboarding](ops/concelier-msrc-operations.md)**
### Legal & licence
- **31[Legal & Quota FAQ](29_LEGAL_FAQ_QUOTA.md)**
</details>

View File

@@ -2,11 +2,14 @@
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|----|--------|----------|------------|-------------|---------------|
| DOC7.README-INDEX | DONE (2025-10-17) | Docs Guild | — | Refresh index docs (docs/README.md + root README) after architecture dossier split and Offline Kit overhaul. | ✅ ToC reflects new component architecture docs; ✅ root README highlights updated doc set; ✅ Offline Kit guide linked correctly. |
| DOC4.AUTH-PDG | REVIEW | Docs Guild, Plugin Team | PLG6.DOC | Copy-edit `docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md`, export lifecycle diagram, add LDAP RFC cross-link. | ✅ PR merged with polish; ✅ Diagram committed; ✅ Slack handoff posted. |
| DOC1.AUTH | DONE (2025-10-12) | Docs Guild, Authority Core | CORE5B.DOC | Draft `docs/11_AUTHORITY.md` covering architecture, configuration, bootstrap flows. | ✅ Architecture + config sections approved by Core; ✅ Samples reference latest options; ✅ Offline note added. |
| DOC3.Feedser-Authority | DONE (2025-10-12) | Docs Guild, DevEx | FSR4 | Polish operator/runbook sections (DOC3/DOC5) to document Feedser authority rollout, bypass logging, and enforcement checklist. | ✅ DOC3/DOC5 updated with audit runbook references; ✅ enforcement deadline highlighted; ✅ Docs guild sign-off. |
| DOC5.Feedser-Runbook | DONE (2025-10-12) | Docs Guild | DOC3.Feedser-Authority | Produce dedicated Feedser authority audit runbook covering log fields, monitoring recommendations, and troubleshooting steps. | ✅ Runbook published; ✅ linked from DOC3/DOC5; ✅ alerting guidance included. |
| FEEDDOCS-DOCS-05-001 | DONE (2025-10-11) | Docs Guild | FEEDMERGE-ENGINE-04-001, FEEDMERGE-ENGINE-04-002 | Publish Feedser conflict resolution runbook covering precedence workflow, merge-event auditing, and Sprint 3 metrics. | ✅ `docs/ops/feedser-conflict-resolution.md` committed; ✅ metrics/log tables align with latest merge code; ✅ Ops alert guidance handed to Feedser team. |
| FEEDDOCS-DOCS-05-002 | TODO | Docs Guild, Feedser Ops | FEEDDOCS-DOCS-05-001 | Capture ops sign-off: circulate conflict runbook, tune alert thresholds, and document rollout decisions in change log. | ✅ Ops review recorded; ✅ alert thresholds finalised using `docs/ops/feedser-authority-audit-runbook.md`; ✅ change-log entry linked from runbook once GHSA/NVD/OSV regression fixtures land. |
| DOC3.Concelier-Authority | DONE (2025-10-12) | Docs Guild, DevEx | FSR4 | Polish operator/runbook sections (DOC3/DOC5) to document Concelier authority rollout, bypass logging, and enforcement checklist. | ✅ DOC3/DOC5 updated with audit runbook references; ✅ enforcement deadline highlighted; ✅ Docs guild sign-off. |
| DOC5.Concelier-Runbook | DONE (2025-10-12) | Docs Guild | DOC3.Concelier-Authority | Produce dedicated Concelier authority audit runbook covering log fields, monitoring recommendations, and troubleshooting steps. | ✅ Runbook published; ✅ linked from DOC3/DOC5; ✅ alerting guidance included. |
| FEEDDOCS-DOCS-05-001 | DONE (2025-10-11) | Docs Guild | FEEDMERGE-ENGINE-04-001, FEEDMERGE-ENGINE-04-002 | Publish Concelier conflict resolution runbook covering precedence workflow, merge-event auditing, and Sprint 3 metrics. | ✅ `docs/ops/concelier-conflict-resolution.md` committed; ✅ metrics/log tables align with latest merge code; ✅ Ops alert guidance handed to Concelier team. |
| FEEDDOCS-DOCS-05-002 | DONE (2025-10-16) | Docs Guild, Concelier Ops | FEEDDOCS-DOCS-05-001 | Ops sign-off captured: conflict runbook circulated, alert thresholds tuned, and rollout decisions documented in change log. | ✅ Ops review recorded; ✅ alert thresholds finalised using `docs/ops/concelier-authority-audit-runbook.md`; ✅ change-log entry linked from runbook once GHSA/NVD/OSV regression fixtures land. |
> Update statuses (TODO/DOING/REVIEW/DONE/BLOCKED) as progress changes. Keep guides in sync with configuration samples under `etc/`.
> Remark (2025-10-13, DOC4.AUTH-PDG): Rate limit guide published (`docs/security/rate-limits.md`) and handed to plugin docs team for diagram uplift once PLG6.DIAGRAM lands.

View File

@@ -0,0 +1,27 @@
<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
<channel>
<title>CISA ICS Advisories</title>
<item>
<title>ICSA-25-123-01: Example ICS Advisory</title>
<link>https://www.cisa.gov/news-events/ics-advisories/icsa-25-123-01</link>
<pubDate>Mon, 13 Oct 2025 12:00:00 GMT</pubDate>
<description><![CDATA[
<p><strong>Vendor:</strong> Example Corp</p>
<p><strong>Products:</strong> ControlSuite 4.2</p>
<p>CVE-2024-12345 allows remote code execution.</p>
<p><a href="https://example.com/security/icsa-25-123-01.pdf">Download PDF</a></p>
]]></description>
</item>
<item>
<title>ICSMA-25-045-01: Example Medical Advisory</title>
<link>https://www.cisa.gov/news-events/ics-medical-advisories/icsma-25-045-01</link>
<pubDate>Tue, 14 Oct 2025 09:30:00 GMT</pubDate>
<description><![CDATA[
<p><strong>Vendor:</strong> HealthTech</p>
<p><strong>Products:</strong> InfusionManager 2.1</p>
<p>Multiple vulnerabilities including CVE-2025-11111 and CVE-2025-22222.</p>
]]></description>
</item>
</channel>
</rss>

View File

@@ -101,5 +101,5 @@
<!-- Notes -->
<text class="note" x="40" y="210">429 path → add `authority.client_id`, `authority.remote_ip` tags for dashboards.</text>
<text class="note" x="40" y="240">Lockout path → reuse precedence strategy from Feedser dedup (see DEDUP_CONFLICTS_RESOLUTION_ALGO.md).</text>
<text class="note" x="40" y="240">Lockout path → reuse precedence strategy from Concelier dedup (see DEDUP_CONFLICTS_RESOLUTION_ALGO.md).</text>
</svg>

Before

Width:  |  Height:  |  Size: 4.8 KiB

After

Width:  |  Height:  |  Size: 4.8 KiB

View File

@@ -1,4 +1,4 @@
# Feedser Connector Research 2025-10-11
# Concelier Connector Research 2025-10-11
Snapshot of direct network checks performed on 2025-10-11 (UTC) for the national/vendor connectors in scope. Use alongside each modules `TASKS.md` notes.
@@ -7,24 +7,28 @@ Snapshot of direct network checks performed on 2025-10-11 (UTC) for the national
- Next actions: prototype `SocketsHttpHandler` settings (`RequestVersionOrLower`, allow fallback to relay), capture successful headers from partner vantage (need retention + cache semantics), and keep `FEEDCONN-SHARED-HTTP2-001` open for downgrade work.
## CCCS (Canada)
- RSS endpoint (`https://cyber.gc.ca/api/cccs/rss/v1/get?...`) 301s to Atom feed (`/api/cccs/atom/v1/get?...`) with 50-entry window, HTML-heavy `<content>` fields, and no cache headers.
- Next actions: enumerate additional `feed` query values, sanitise inline HTML for DTO storage, and track retention depth via HTML pagination (`?page=`).
- JSON endpoint (`https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=<lang>&content_type=cccs_threat`) returns ~5100 records per language; `page=<n>` still works for segmented pulls and the earliest `date_created` seen is 20180608 (EN) / 20180608 (FR). Use an explicit `User-Agent` to avoid 403 responses.
- Follow-up: telemetry, sanitiser coverage, and backfill procedures are documented in `docs/ops/concelier-cccs-operations.md` (20251015). Adjust `maxEntriesPerFetch` when performing historical sweeps so cursor state remains responsive.
## CERT-Bund (Germany)
- `https://wid.cert-bund.de/content/public/securityAdvisory/rss` responds 200 without cookies (250-item window, German taxonomy). Detail links load an Angular SPA that fetches JSON behind session cookies.
- Next actions: script SPA cookie/bootstrap, discover JSON detail endpoint, and capture advisory schema for parser planning.
- `https://wid.cert-bund.de/content/public/securityAdvisory/rss` responds 200 without cookies (250-item window, German taxonomy). Detail links load an Angular SPA that fetches JSON behind the bootstrap session.
- Confirmed `GET https://wid.cert-bund.de/portal/api/securityadvisory?name=<WID-SEC-…>` returns JSON once the portal cookie container is primed; payload includes severity, CVEs, products, and references used by the connector fixtures.
- Historical advisories accessible through the SPA search/export endpoints once the `XSRF-TOKEN` cookie (exposed via `GET /portal/api/security/csrf`) is supplied with the `X-XSRF-TOKEN` header:
- `POST /portal/api/securityadvisory/search` (`{"page":N,"size":100,"sort":["published,desc"]}`) pages data back to 2014.
- `GET /portal/api/securityadvisory/export?format=json&from=YYYY-MM-DD` emits JSON bundles suitable for Offline Kit mirrors.
- Locale note: content is German-only; Concelier preserves `language=de` and Docs will publish a CERT-Bund glossary so operators can bridge terminology without machine translation.
## KISA / KNVD (Korea)
- `https://knvd.krcert.or.kr/rss/securityInfo.do` and `/rss/securityNotice.do` return UTF-8 RSS (10-item window) with `detailDos.do?IDX=` links. No cookies required for feed fetch.
- Next actions: trace SPA detail requests to identify JSON endpoints, normalise Hangul content, and finalise localisation plan.
- Detail SPA calls resolve to `rssDetailData.do?IDX=` JSON payloads; connector fetches those directly, sanitises HTML, and records Hangul metadata (NFC). See `docs/dev/kisa_connector_notes.md` for telemetry + localisation guidance.
## BDU (Russia / FSTEC)
- Candidate endpoints (`https://bdu.fstec.ru/component/rsform/form/7-bdu?format=xml/json`) return 403/404; TLS chain requires Russian Trusted Sub CA and WAF expects additional headers.
- Next actions: acquire official PEM chain, point `feedser:httpClients:source.bdu:trustedRootPaths` (or `feedser:sources:bdu:http:trustedRootPaths`) at the Offline Kit PEM, keep `allowInvalidCertificates=false`, script session bootstrap, then capture RSS/HTML schema for parser work.
- Next actions: acquire official PEM chain, point `concelier:httpClients:source.bdu:trustedRootPaths` (or `concelier:sources:bdu:http:trustedRootPaths`) at the Offline Kit PEM, keep `allowInvalidCertificates=false`, script session bootstrap, then capture RSS/HTML schema for parser work.
## NKTsKI / cert.gov.ru (Russia)
- `https://cert.gov.ru/rss/advisories.xml` served via Bitrix returns 403/404 even with `Accept-Language: ru-RU`; TLS chain also requires Russian trust anchors.
- Next actions: source trust store, configure `feedser:httpClients:source.nkcki:trustedRootPaths` (Offline Kit root via `feedser:offline:root`), prepare proxy fallback, and once accessible document taxonomy/retention plus attachment handling.
- Next actions: source trust store, configure `concelier:httpClients:source.nkcki:trustedRootPaths` (Offline Kit root via `concelier:offline:root`), prepare proxy fallback, and once accessible document taxonomy/retention plus attachment handling.
## CISA ICS (United States)
- `curl -I https://www.cisa.gov/cybersecurity-advisories/ics-advisories.xml` returns HTTP 403 + `x-reference-error` (Akamai). Same for legacy feed paths.

View File

@@ -0,0 +1,220 @@
# Excititor Connector Packaging Guide
> **Audience:** teams implementing new Excititor provider plugins (CSAF feeds,
> OpenVEX attestations, etc.)
> **Prerequisites:** read `docs/ARCHITECTURE_EXCITITOR.md` and the module
> `AGENTS.md` in `src/StellaOps.Excititor.Connectors.Abstractions/`.
The Excititor connector SDK gives you:
- `VexConnectorBase` deterministic logging, SHA256 helpers, time provider.
- `VexConnectorOptionsBinder` strongly typed YAML/JSON configuration binding.
- `IVexConnectorOptionsValidator<T>` custom validation hooks (offline defaults, auth invariants).
- `VexConnectorDescriptor` & metadata helpers for consistent telemetry.
This guide explains how to package a connector so the Excititor Worker/WebService
can load it via the plugin host.
---
## 1. Project layout
Start from the template under
`docs/dev/templates/excititor-connector/`. It contains:
```
Excititor.MyConnector/
├── src/
│ ├── Excititor.MyConnector.csproj
│ ├── MyConnectorOptions.cs
│ ├── MyConnector.cs
│ └── MyConnectorPlugin.cs
└── manifest/
└── connector.manifest.yaml
```
Key points:
- Target `net10.0`, enable `TreatWarningsAsErrors`, reference the
`StellaOps.Excititor.Connectors.Abstractions` project (or NuGet once published).
- Keep project ID prefix `StellaOps.Excititor.Connectors.<Provider>` so the
plugin loader can discover it with the default search pattern.
### 1.1 csproj snippet
```xml
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\src\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
</ItemGroup>
</Project>
```
Adjust the `ProjectReference` for your checkout (or switch to a NuGet package
once published).
---
## 2. Implement the connector
1. **Options model** create an options POCO with data-annotation attributes.
Bind it via `VexConnectorOptionsBinder.Bind<TOptions>` in your connector
constructor or `ValidateAsync`.
2. **Validator** implement `IVexConnectorOptionsValidator<TOptions>` to add
complex checks (e.g., ensure both `clientId` and `clientSecret` are present).
3. **Connector** inherit from `VexConnectorBase`. Implement:
- `ValidateAsync` run binder/validators, log configuration summary.
- `FetchAsync` stream raw documents to `context.RawSink`.
- `NormalizeAsync` convert raw documents into `VexClaimBatch` via
format-specific normalizers (`context.Normalizers`).
4. **Plugin adapter** expose the connector via a plugin entry point so the
host can instantiate it.
### 2.1 Options binding example
```csharp
public sealed class MyConnectorOptions
{
[Required]
[Url]
public string CatalogUri { get; set; } = default!;
[Required]
public string ApiKey { get; set; } = default!;
[Range(1, 64)]
public int MaxParallelRequests { get; set; } = 4;
}
public sealed class MyConnectorOptionsValidator : IVexConnectorOptionsValidator<MyConnectorOptions>
{
public void Validate(VexConnectorDescriptor descriptor, MyConnectorOptions options, IList<string> errors)
{
if (!options.CatalogUri.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
{
errors.Add("CatalogUri must use HTTPS.");
}
}
}
```
Bind inside the connector:
```csharp
private readonly MyConnectorOptions _options;
public MyConnector(VexConnectorDescriptor descriptor, ILogger<MyConnector> logger, TimeProvider timeProvider)
: base(descriptor, logger, timeProvider)
{
// `settings` comes from the orchestrator; validators registered via DI.
_options = VexConnectorOptionsBinder.Bind<MyConnectorOptions>(
descriptor,
VexConnectorSettings.Empty,
validators: new[] { new MyConnectorOptionsValidator() });
}
```
Replace `VexConnectorSettings.Empty` with the actual settings from context
inside `ValidateAsync`.
---
## 3. Plugin adapter & manifest
Create a simple plugin class that implements
`StellaOps.Plugin.IConnectorPlugin`. The Worker/WebService plugin host uses
this contract today.
```csharp
public sealed class MyConnectorPlugin : IConnectorPlugin
{
private static readonly VexConnectorDescriptor Descriptor =
new("excititor:my-provider", VexProviderKind.Vendor, "My Provider VEX");
public string Name => Descriptor.DisplayName;
public bool IsAvailable(IServiceProvider services) => true; // inject feature flags if needed
public IFeedConnector Create(IServiceProvider services)
{
var logger = services.GetRequiredService<ILogger<MyConnector>>();
var timeProvider = services.GetRequiredService<TimeProvider>();
return new MyConnector(Descriptor, logger, timeProvider);
}
}
```
> **Note:** the Excititor Worker currently instantiates connectors through the
> shared `IConnectorPlugin` contract. Once a dedicated Excititor plugin interface
> lands you simply swap the base interface; the descriptor/connector code
> remains unchanged.
Provide a manifest describing the assembly for operational tooling:
```yaml
# manifest/connector.manifest.yaml
id: excititor-my-provider
assembly: StellaOps.Excititor.Connectors.MyProvider.dll
entryPoint: StellaOps.Excititor.Connectors.MyProvider.MyConnectorPlugin
description: >
Official VEX feed for ExampleCorp products (CSAF JSON, daily updates).
tags:
- excititor
- csaf
- vendor
```
Store manifests under `/opt/stella/excititor/plugins/<connector>/manifest/` in
production so the deployment tooling can inventory and verify plugins.
---
## 4. Packaging workflow
1. `dotnet publish -c Release` → copy the published DLLs to
`/opt/stella/excititor/plugins/<Provider>/`.
2. Place `connector.manifest.yaml` next to the binaries.
3. Restart the Excititor Worker or WebService (hot reload not supported yet).
4. Verify logs: `VEX-ConnectorLoader` should list the connector descriptor.
### 4.1 Offline kits
- Add the connector folder (binaries + manifest) to the Offline Kit bundle.
- Include a `settings.sample.yaml` demonstrating offline-friendly defaults.
- Document any external dependencies (e.g., SHA mirrors) in the manifest `notes`
field.
---
## 5. Testing checklist
- Unit tests around options binding & validators.
- Integration tests (future `StellaOps.Excititor.Connectors.Abstractions.Tests`)
verifying deterministic logging scopes:
`logger.BeginScope` should produce `vex.connector.id`, `vex.connector.kind`,
and `vex.connector.operation`.
- Deterministic SHA tests: repeated `CreateRawDocument` calls with identical
content must return the same digest.
---
## 6. Reference template
See `docs/dev/templates/excititor-connector/` for the full quickstart including:
- Sample options class + validator.
- Connector implementation inheriting from `VexConnectorBase`.
- Plugin adapter + manifest.
Copy the directory, rename namespaces/IDs, then iterate on provider-specific
logic.
---
*Last updated: 2025-10-17*

View File

@@ -3,7 +3,7 @@
> **Status:** Updated 2025-10-11 (AUTHPLUG-DOCS-01-001) with lifecycle + limiter diagrams and refreshed rate-limit guidance aligned to PLG6 acceptance criteria.
## 1. Overview
Authority plug-ins extend the **StellaOps Authority** service with custom identity providers, credential stores, and client-management logic. Unlike Feedser plug-ins (which ingest or export advisories), Authority plug-ins participate directly in authentication flows:
Authority plug-ins extend the **StellaOps Authority** service with custom identity providers, credential stores, and client-management logic. Unlike Concelier plug-ins (which ingest or export advisories), Authority plug-ins participate directly in authentication flows:
- **Use cases:** integrate corporate directories (LDAP/AD), delegate to external IDPs, enforce bespoke password/lockout policies, or add client provisioning automation.
- **Constraints:** plug-ins load only during service start (no hot-reload), must function without outbound internet access, and must emit deterministic results for identical configuration and input data.
@@ -44,6 +44,8 @@ Capability flags let the host reason about what your plug-in supports:
**Configuration path normalisation:** Manifest-relative paths (e.g., `tokenSigning.keyDirectory: "../keys"`) are resolved against the YAML file location and environment variables are expanded before validation. Plug-ins should expect to receive an absolute, canonical path when options are injected.
**Password policy guardrails:** The Standard registrar logs a warning when a plug-in weakens the default password policy (minimum length or required character classes). Keep overrides at least as strong as the compiled defaults—operators treat the warning as an actionable security deviation.
## 4. Project Scaffold
- Target **.NET 10 preview**, enable nullable, treat warnings as errors, and mark Authority plug-ins with `<IsAuthorityPlugin>true</IsAuthorityPlugin>`.
- Minimum references:

View File

@@ -1,6 +1,6 @@
# StellaOps Auth Client — Integration Guide
> **Status:** Drafted 2025-10-10 as part of LIB5. Consumer teams (Feedser, CLI, Agent) should review before wiring the new options into their configuration surfaces.
> **Status:** Drafted 2025-10-10 as part of LIB5. Consumer teams (Concelier, CLI, Agent) should review before wiring the new options into their configuration surfaces.
The `StellaOps.Auth.Client` library provides a resilient OpenID Connect client for services and tools that talk to **StellaOps Authority**. LIB5 introduced configurable HTTP retry/backoff policies and an offline-fallback window so downstream components stay deterministic even when Authority is briefly unavailable.
@@ -14,7 +14,7 @@ services.AddStellaOpsAuthClient(options =>
options.Authority = configuration["StellaOps:Authority:Url"]!;
options.ClientId = configuration["StellaOps:Authority:ClientId"]!;
options.ClientSecret = configuration["StellaOps:Authority:ClientSecret"];
options.DefaultScopes.Add("feedser.jobs.trigger");
options.DefaultScopes.Add("concelier.jobs.trigger");
options.EnableRetries = true;
options.RetryDelays.Clear();
@@ -53,7 +53,7 @@ Suggested configuration keys (coordinate with consuming teams before finalising)
StellaOps:
Authority:
Url: "https://authority.stella-ops.local"
ClientId: "feedser"
ClientId: "concelier"
ClientSecret: "change-me"
AuthClient:
EnableRetries: true
@@ -72,7 +72,7 @@ STELLAOPS__AUTHORITY__AUTHCLIENT__RETRYDELAYS__0=00:00:02
STELLAOPS__AUTHORITY__AUTHCLIENT__OFFLINECACHETOLERANCE=00:05:00
```
CLI and Feedser teams should expose these knobs once they adopt the auth client.
CLI and Concelier teams should expose these knobs once they adopt the auth client.
## 4. Testing recommendations
@@ -88,4 +88,4 @@ CLI and Feedser teams should expose these knobs once they adopt the auth client.
- [ ] Extend smoke tests to cover Authority outage scenarios.
- [ ] Coordinate with Docs Guild so user-facing quickstarts reference the new knobs.
Once Feedser and CLI integrate these changes, we can mark LIB5 **DONE**; further packaging work is deferred until the backlog reintroduces it.
Once Concelier and CLI integrate these changes, we can mark LIB5 **DONE**; further packaging work is deferred until the backlog reintroduces it.

View File

@@ -1,37 +1,45 @@
# Feedser Fixture Maintenance
# Concelier Fixture Maintenance
Feedser uses a handful of deterministic fixtures to keep connector regressions in check. This guide lists the
Concelier uses a handful of deterministic fixtures to keep connector regressions in check. This guide lists the
fixture sets, where they live, and how to regenerate them safely.
---
## GHSA ↔ OSV parity fixtures
- **Location:** `src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.*.json`
- **Location:** `src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/osv-ghsa.*.json`
- **Purpose:** Exercised by `OsvGhsaParityRegressionTests` to ensure OSV + GHSA outputs stay aligned on aliases,
ranges, references, and credits.
- **Regeneration:** Either run the test harness with online regeneration (`UPDATE_PARITY_FIXTURES=1 dotnet test src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj`)
- **Regeneration:** Either run the test harness with online regeneration (`UPDATE_PARITY_FIXTURES=1 dotnet test src/StellaOps.Concelier.Connector.Osv.Tests/StellaOps.Concelier.Connector.Osv.Tests.csproj`)
or execute the fixture updater (`dotnet run --project tools/FixtureUpdater/FixtureUpdater.csproj`). Both paths
normalise timestamps and canonical ordering.
- **SemVer provenance:** The regenerated fixtures should show `normalizedVersions[].notes` in the
`osv:{ecosystem}:{advisoryId}:{identifier}` shape emitted by `SemVerRangeRuleBuilder`. Confirm the
constraints and notes line up with GHSA/NVD composites before committing.
- **Verification:** Inspect the diff, then re-run `dotnet test src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj` to confirm parity.
- **Verification:** Inspect the diff, then re-run `dotnet test src/StellaOps.Concelier.Connector.Osv.Tests/StellaOps.Concelier.Connector.Osv.Tests.csproj` to confirm parity.
## GHSA credit parity fixtures
- **Location:** `src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/credit-parity.{ghsa,osv,nvd}.json`
- **Location:** `src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/credit-parity.{ghsa,osv,nvd}.json`
- **Purpose:** Exercised by `GhsaCreditParityRegressionTests` to guarantee GHSA/NVD/OSV acknowledgements remain in lockstep.
- **Regeneration:** `dotnet run --project tools/FixtureUpdater/FixtureUpdater.csproj` rewrites all three canonical snapshots.
- **Verification:** `dotnet test src/StellaOps.Feedser.Source.Ghsa.Tests/StellaOps.Feedser.Source.Ghsa.Tests.csproj`.
- **Verification:** `dotnet test src/StellaOps.Concelier.Connector.Ghsa.Tests/StellaOps.Concelier.Connector.Ghsa.Tests.csproj`.
> Always commit fixture changes together with the code that motivated them and reference the regression test that guards the behaviour.
## Apple security update fixtures
- **Location:** `src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/*.html` and `.expected.json`.
- **Location:** `src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/*.html` and `.expected.json`.
- **Purpose:** Exercised by `AppleLiveRegressionTests` to guarantee the Apple HTML parser and mapper stay deterministic while covering Rapid Security Responses and multi-device advisories.
- **Regeneration:** Use the helper scripts (`scripts/update-apple-fixtures.sh` or `scripts/update-apple-fixtures.ps1`). They export `UPDATE_APPLE_FIXTURES=1`, propagate the flag through `WSLENV`, touch `.update-apple-fixtures`, and then run the Apple test project. This keeps WSL/VSCode test invocations in sync while the refresh workflow fetches live Apple support pages, sanitises them, and rewrites both the HTML and expected DTO snapshots with normalised ordering.
- **Verification:** Inspect the generated diffs and re-run `dotnet test src/StellaOps.Feedser.Source.Vndr.Apple.Tests/StellaOps.Feedser.Source.Vndr.Apple.Tests.csproj` without the env var to confirm determinism.
- **Verification:** Inspect the generated diffs and re-run `dotnet test src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/StellaOps.Concelier.Connector.Vndr.Apple.Tests.csproj` without the env var to confirm determinism.
> **Tip for other connector owners:** mirror the sentinel + `WSLENV` pattern (`touch .update-<connector>-fixtures`, append the env var via `WSLENV`) when you add fixture refresh scripts so contributors running under WSL inherit the regeneration flag automatically.
## KISA advisory fixtures
- **Location:** `src/StellaOps.Concelier.Connector.Kisa.Tests/Fixtures/kisa-{feed,detail}.(xml|json)`
- **Purpose:** Used by `KisaConnectorTests` to verify Hangul-aware fetch → parse → map flows and to assert telemetry counters stay wired.
- **Regeneration:** `UPDATE_KISA_FIXTURES=1 dotnet test src/StellaOps.Concelier.Connector.Kisa.Tests/StellaOps.Concelier.Connector.Kisa.Tests.csproj`
- **Verification:** Re-run the same test suite without the env var; confirm advisory content remains NFC-normalised and HTML is sanitised. Metrics assertions will fail if counters drift.
- **Localisation note:** RSS `category` values (e.g. `취약점정보`) remain in Hangul—do not translate them in fixtures; they feed directly into metrics/log tags.

View File

@@ -0,0 +1,45 @@
# KISA Connector Observability & Localisation
The KISA/KNVD connector now ships with structured telemetry, richer logging, and a localisation brief so Docs/QA can extend operator material without reverse-engineering the source.
## Telemetry counters
All metrics are emitted from `KisaDiagnostics` (`Meter` name `StellaOps.Concelier.Connector.Kisa`).
| Metric | Description | Tags |
| --- | --- | --- |
| `kisa.feed.attempts` | RSS fetch attempts per scheduled job. | — |
| `kisa.feed.success` | Successful RSS fetches (increments even when no new items). | — |
| `kisa.feed.failures` | RSS fetch failures. | `reason` (exception type) |
| `kisa.feed.items` | Number of items returned by the RSS window. | — |
| `kisa.detail.attempts` | Advisory detail fetch attempts. | `category` (Hangul category from RSS) |
| `kisa.detail.success` | Detail payloads fetched and persisted. | `category` |
| `kisa.detail.unchanged` | HTTP 304 responses reused from cache. | `category` |
| `kisa.detail.failures` | Detail fetch failures or empty payloads. | `category`, `reason` |
| `kisa.parse.attempts` | Documents pulled from Mongo for parsing. | `category` |
| `kisa.parse.success` | Documents parsed into DTOs. | `category` |
| `kisa.parse.failures` | Download or JSON parse failures. | `category`, `reason` |
| `kisa.map.success` | Canonical advisories persisted. | `severity` (e.g. `High`, `unknown`) |
| `kisa.map.failures` | Mapping or DTO hydration failures. | `severity`, `reason` |
| `kisa.cursor.updates` | Published cursor advanced after ingest. | — |
> `category` tags surface the original Hangul labels (for example `취약점정보`), normalised to NFC. Downstream dashboards should render them as-is; do not transliterate or trim.
## Logging patterns
- `Information` level summary when the RSS feed completes (`ItemCount`), on each persisted detail document (IDX, category, documentId), and when a canonical advisory is written (IDX/severity).
- `Debug` level logs capture cache hits (304) and cursor movements (`Published` timestamp).
- `Warning` level emits when a document or DTO is missing so operators can correlate with parse/map counters.
- `Error` level retains exception context for feed/detail/parse/map failures; state repository backoffs are still applied.
The messages use structured properties (`Idx`, `Category`, `DocumentId`, `Severity`) so Grafana/Loki dashboards can filter without regex.
## Localisation notes for Docs & QA
- Hangul fields (`title`, `summary`, `category`, `reference.label`, product vendor/name) are normalised to NFC before storage. Sample category `취약점정보` roughly translates to “vulnerability information”.
- Advisory HTML is sanitised via `HtmlContentSanitizer`, stripping script/style while preserving inline anchors for translation pipelines.
- Metrics carry Hangul `category` tags and logging keeps Hangul strings intact; this ensures air-gapped operators can validate native-language content without relying on MT.
- Fixtures live under `src/StellaOps.Concelier.Connector.Kisa.Tests/Fixtures/`. Regenerate with `UPDATE_KISA_FIXTURES=1 dotnet test src/StellaOps.Concelier.Connector.Kisa.Tests/StellaOps.Concelier.Connector.Kisa.Tests.csproj`.
- The regression suite asserts canonical mapping, state cleanup, and telemetry counters (`KisaConnectorTests.Telemetry_RecordsMetrics`) so QA can track instrumentation drift.
For operator docs, link to this brief when documenting Hangul handling or counter dashboards so localisation reviewers have a single reference point.

View File

@@ -1,4 +1,4 @@
# Feedser SemVer Merge Playbook (Sprint 12)
# Concelier SemVer Merge Playbook (Sprint 12)
This playbook describes how the merge layer and connector teams should emit the new SemVer primitives introduced in Sprint12, how those primitives become normalized version rules, and how downstream jobs query them deterministically.
@@ -8,7 +8,7 @@ This playbook describes how the merge layer and connector teams should emit the
- `NormalizedVersionRule` documents the analytics-friendly projection of each `AffectedPackage` coverage entry and is persisted alongside legacy `versionRanges`.
- `AdvisoryProvenance.decisionReason` records whether merge resolution favored precedence, freshness, or a tie-breaker comparison.
See `src/StellaOps.Feedser.Models/CANONICAL_RECORDS.md` for the full schema and field descriptions.
See `src/StellaOps.Concelier.Models/CANONICAL_RECORDS.md` for the full schema and field descriptions.
## 2. Mapper pattern
@@ -150,5 +150,5 @@ If a new scheme is required (for example, `apple.build` or `ios.semver`), raise
## 9. Observability signals
- `feedser.merge.normalized_rules` (counter, tags: `package_type`, `scheme`) increments once per normalized rule retained after precedence merge.
- `feedser.merge.normalized_rules_missing` (counter, tags: `package_type`) increments when a merged package still carries version ranges but no normalized rules; watch for spikes to catch connectors that have not emitted normalized arrays yet.
- `concelier.merge.normalized_rules` (counter, tags: `package_type`, `scheme`) increments once per normalized rule retained after precedence merge.
- `concelier.merge.normalized_rules_missing` (counter, tags: `package_type`) increments when a merged package still carries version ranges but no normalized rules; watch for spikes to catch connectors that have not emitted normalized arrays yet.

View File

@@ -2,11 +2,11 @@
This guide complements the Sprint12 normalized versions rollout. It documents recommended indexes and aggregation patterns for querying `AffectedPackage.normalizedVersions`.
For a field-by-field look at how normalized rules persist in MongoDB (including provenance metadata), see Section8 of the [Feedser SemVer Merge Playbook](merge_semver_playbook.md).
For a field-by-field look at how normalized rules persist in MongoDB (including provenance metadata), see Section8 of the [Concelier SemVer Merge Playbook](merge_semver_playbook.md).
## 1. Recommended indexes
When `feedser.storage.enableSemVerStyle` is enabled, advisories expose a flattened
When `concelier.storage.enableSemVerStyle` is enabled, advisories expose a flattened
`normalizedVersions` array at the document root. Create these indexes in `mongosh`
after the migration completes (adjust collection name if you use a prefix):
@@ -105,4 +105,4 @@ Use this to confirm the merge dedupe logic keeps only one normalized rule per un
- [ ] Notify downstream services that consume advisory snapshots about the new `normalizedVersions` array.
- [ ] Update export fixtures once dedupe verification passes.
Additional background and mapper examples live in [Feedser SemVer Merge Playbook](merge_semver_playbook.md).
Additional background and mapper examples live in [Concelier SemVer Merge Playbook](merge_semver_playbook.md).

View File

@@ -1,11 +1,11 @@
# Normalized Versions Rollout Dashboard (Sprint 2 Feedser)
# Normalized Versions Rollout Dashboard (Sprint 2 Concelier)
_Status date: 2025-10-12 17:05 UTC_
This dashboard tracks connector readiness for emitting `AffectedPackage.NormalizedVersions` arrays and highlights upcoming coordination checkpoints. Use it alongside:
- [`src/StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md`](../../src/StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md) for detailed guidance and timelines.
- [Feedser SemVer Merge Playbook](merge_semver_playbook.md) §8 for persisted Mongo document shapes.
- [`src/StellaOps.Concelier.Merge/RANGE_PRIMITIVES_COORDINATION.md`](../../src/StellaOps.Concelier.Merge/RANGE_PRIMITIVES_COORDINATION.md) for detailed guidance and timelines.
- [Concelier SemVer Merge Playbook](merge_semver_playbook.md) §8 for persisted Mongo document shapes.
- [Normalized Versions Query Guide](mongo_indices.md) for index/query validation steps.
## Key milestones
@@ -18,30 +18,34 @@ This dashboard tracks connector readiness for emitting `AffectedPackage.Normaliz
| Connector | Owner team | Normalized versions status | Last update | Next action / link |
|-----------|------------|---------------------------|-------------|--------------------|
| Acsc | BE-Conn-ACSC | ❌ Not started mapper pending | 2025-10-11 | Design DTOs + mapper with normalized rule array; see `src/StellaOps.Feedser.Source.Acsc/TASKS.md`. |
| Acsc | BE-Conn-ACSC | ❌ Not started mapper pending | 2025-10-11 | Design DTOs + mapper with normalized rule array; see `src/StellaOps.Concelier.Connector.Acsc/TASKS.md`. |
| Cccs | BE-Conn-CCCS | ❌ Not started mapper pending | 2025-10-11 | Add normalized SemVer array in canonical mapper; coordinate fixtures per `TASKS.md`. |
| CertBund | BE-Conn-CERTBUND | ❌ Not started mapper pending | 2025-10-11 | Capture firmware-style ranges; emit normalized payload; `src/StellaOps.Feedser.Source.CertBund/TASKS.md`. |
| CertCc | BE-Conn-CERTCC | ⚠️ In progress fetch pipeline DOING | 2025-10-11 | Implement VINCE mapper with SemVer/NEVRA rules; unblock snapshot regeneration; `src/StellaOps.Feedser.Source.CertCc/TASKS.md`. |
| Kev | BE-Conn-KEV | ✅ Normalized catalog/due-date rules verified | 2025-10-12 | Fixtures reconfirmed via `dotnet test src/StellaOps.Feedser.Source.Kev.Tests`; `src/StellaOps.Feedser.Source.Kev/TASKS.md`. |
| Cve | BE-Conn-CVE | ✅ Normalized SemVer rules verified | 2025-10-12 | Snapshot parity green (`dotnet test src/StellaOps.Feedser.Source.Cve.Tests`); `src/StellaOps.Feedser.Source.Cve/TASKS.md`. |
| Ghsa | BE-Conn-GHSA | ⚠️ DOING normalized rollout task active | 2025-10-11 18:45 UTC | Wire `SemVerRangeRuleBuilder` + refresh fixtures; `src/StellaOps.Feedser.Source.Ghsa/TASKS.md`. |
| Osv | BE-Conn-OSV | ✅ SemVer mapper & parity fixtures verified | 2025-10-12 | GHSA parity regression passing (`dotnet test src/StellaOps.Feedser.Source.Osv.Tests`); `src/StellaOps.Feedser.Source.Osv/TASKS.md`. |
| Ics.Cisa | BE-Conn-ICS-CISA | ❌ Not started mapper TODO | 2025-10-11 | Plan SemVer/firmware scheme selection; `src/StellaOps.Feedser.Source.Ics.Cisa/TASKS.md`. |
| Kisa | BE-Conn-KISA | ❌ Not started mapper TODO | 2025-10-11 | Localisation-aware mapper with normalized rules; `src/StellaOps.Feedser.Source.Kisa/TASKS.md`. |
| Ru.Bdu | BE-Conn-BDU | ❌ Not started mapper TODO | 2025-10-11 | Emit normalized ranges, capture provenance; `src/StellaOps.Feedser.Source.Ru.Bdu/TASKS.md`. |
| Ru.Nkcki | BE-Conn-Nkcki | ❌ Not started mapper TODO | 2025-10-11 | Similar to BDU; ensure Cyrillic provenance preserved; `src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md`. |
| Vndr.Apple | BE-Conn-Apple | ✅ Shipped emitting normalized arrays | 2025-10-11 | Continue fixture/tooling work; `src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md`. |
| Vndr.Cisco | BE-Conn-Cisco | ❌ Not started mapper TODO | 2025-10-11 | Decide on scheme (`semver` vs custom) before emitting rules; `src/StellaOps.Feedser.Source.Vndr.Cisco/TASKS.md`. |
| Vndr.Msrc | BE-Conn-MSRC | ❌ Not started mapper TODO | 2025-10-11 | Gather samples, define scheme, emit normalized rules; `src/StellaOps.Feedser.Source.Vndr.Msrc/TASKS.md`. |
| Nvd | BE-Conn-NVD | ⚠️ Needs follow-up mapper complete but normalized array MR pending | 2025-10-11 | Align CVE notes + normalized payload flag; `src/StellaOps.Feedser.Source.Nvd/TASKS.md`. |
| CertBund | BE-Conn-CERTBUND | ✅ Canonical mapper emitting vendor ranges | 2025-10-14 | Normalized vendor range payloads landed alongside telemetry/docs updates; see `src/StellaOps.Concelier.Connector.CertBund/TASKS.md`. |
| CertCc | BE-Conn-CERTCC | ⚠️ In progress fetch pipeline DOING | 2025-10-11 | Implement VINCE mapper with SemVer/NEVRA rules; unblock snapshot regeneration; `src/StellaOps.Concelier.Connector.CertCc/TASKS.md`. |
| Kev | BE-Conn-KEV | ✅ Normalized catalog/due-date rules verified | 2025-10-12 | Fixtures reconfirmed via `dotnet test src/StellaOps.Concelier.Connector.Kev.Tests`; `src/StellaOps.Concelier.Connector.Kev/TASKS.md`. |
| Cve | BE-Conn-CVE | ✅ Normalized SemVer rules verified | 2025-10-12 | Snapshot parity green (`dotnet test src/StellaOps.Concelier.Connector.Cve.Tests`); `src/StellaOps.Concelier.Connector.Cve/TASKS.md`. |
| Ghsa | BE-Conn-GHSA | ⚠️ DOING normalized rollout task active | 2025-10-11 18:45 UTC | Wire `SemVerRangeRuleBuilder` + refresh fixtures; `src/StellaOps.Concelier.Connector.Ghsa/TASKS.md`. |
| Osv | BE-Conn-OSV | ✅ SemVer mapper & parity fixtures verified | 2025-10-12 | GHSA parity regression passing (`dotnet test src/StellaOps.Concelier.Connector.Osv.Tests`); `src/StellaOps.Concelier.Connector.Osv/TASKS.md`. |
| Ics.Cisa | BE-Conn-ICS-CISA | ❌ Not started mapper TODO | 2025-10-11 | Plan SemVer/firmware scheme selection; `src/StellaOps.Concelier.Connector.Ics.Cisa/TASKS.md`. |
| Kisa | BE-Conn-KISA | ✅ Landed 2025-10-14 (mapper + telemetry) | 2025-10-11 | Hangul-aware mapper emits normalized rules; see `docs/dev/kisa_connector_notes.md` for localisation/metric details. |
| Ru.Bdu | BE-Conn-BDU | ✅ Raw scheme emitted | 2025-10-14 | Mapper now writes `ru-bdu.raw` normalized rules with provenance + telemetry; `src/StellaOps.Concelier.Connector.Ru.Bdu/TASKS.md`. |
| Ru.Nkcki | BE-Conn-Nkcki | ❌ Not started mapper TODO | 2025-10-11 | Similar to BDU; ensure Cyrillic provenance preserved; `src/StellaOps.Concelier.Connector.Ru.Nkcki/TASKS.md`. |
| Vndr.Apple | BE-Conn-Apple | ✅ Shipped emitting normalized arrays | 2025-10-11 | Continue fixture/tooling work; `src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md`. |
| Vndr.Cisco | BE-Conn-Cisco | ✅ SemVer + vendor extensions emitted | 2025-10-14 | Connector outputs SemVer primitives with `cisco.productId` notes; see `CiscoMapper` and fixtures for coverage. |
| Vndr.Msrc | BE-Conn-MSRC | ✅ Map + normalized build rules landed | 2025-10-15 | `MsrcMapper` emits `msrc.build` normalized rules with CVRF references; see `src/StellaOps.Concelier.Connector.Vndr.Msrc/TASKS.md`. |
| Nvd | BE-Conn-NVD | ⚠️ Needs follow-up mapper complete but normalized array MR pending | 2025-10-11 | Align CVE notes + normalized payload flag; `src/StellaOps.Concelier.Connector.Nvd/TASKS.md`. |
Legend: ✅ complete, ⚠️ in progress/partial, ❌ not started.
## Monitoring
- Merge now emits `feedser.merge.normalized_rules` (tags: `package_type`, `scheme`) and `feedser.merge.normalized_rules_missing` (tags: `package_type`). Track these counters to confirm normalized arrays land as connectors roll out.
- Merge now emits `concelier.merge.normalized_rules` (tags: `package_type`, `scheme`) and `concelier.merge.normalized_rules_missing` (tags: `package_type`). Track these counters to confirm normalized arrays land as connectors roll out.
- Expect `normalized_rules_missing` to trend toward zero as each connector flips on normalized output. Investigate any sustained counts by checking the corresponding module `TASKS.md`.
## Implementation tips
- When a connector only needs to populate `AffectedPackage.NormalizedVersions` (without reusing range primitives), call `SemVerRangeRuleBuilder.BuildNormalizedRules(rawRange, patchedVersion, note)` to project the normalized rule list directly. This avoids re-wrapping `SemVerRangeBuildResult` instances and keeps provenance notes consistent with the shared builder.
## How to use this dashboard
1. Before opening a connector PR, update the module `TASKS.md` entry and drop a short bullet here (status + timestamp).

View File

@@ -0,0 +1,8 @@
id: excititor-my-provider
assembly: StellaOps.Excititor.Connectors.MyProvider.dll
entryPoint: StellaOps.Excititor.Connectors.MyProvider.MyConnectorPlugin
description: |
Example connector template. Replace metadata before shipping.
tags:
- excititor
- template

View File

@@ -0,0 +1,12 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<!-- Adjust the relative path when copying this template into a repo -->
<ProjectReference Include="..\..\..\..\src\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,72 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Runtime.CompilerServices;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Connectors.MyProvider;
public sealed class MyConnector : VexConnectorBase
{
private readonly IEnumerable<IVexConnectorOptionsValidator<MyConnectorOptions>> _validators;
private MyConnectorOptions? _options;
public MyConnector(VexConnectorDescriptor descriptor, ILogger<MyConnector> logger, TimeProvider timeProvider, IEnumerable<IVexConnectorOptionsValidator<MyConnectorOptions>> validators)
: base(descriptor, logger, timeProvider)
{
_validators = validators;
}
public override ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
{
_options = VexConnectorOptionsBinder.Bind(
Descriptor,
settings,
validators: _validators);
LogConnectorEvent(LogLevel.Information, "validate", "MyConnector configuration loaded.",
new Dictionary<string, object?>
{
["catalogUri"] = _options.CatalogUri,
["maxParallelRequests"] = _options.MaxParallelRequests,
});
return ValueTask.CompletedTask;
}
public override IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, CancellationToken cancellationToken)
{
if (_options is null)
{
throw new InvalidOperationException("Connector not validated.");
}
return FetchInternalAsync(context, cancellationToken);
}
private async IAsyncEnumerable<VexRawDocument> FetchInternalAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken)
{
LogConnectorEvent(LogLevel.Information, "fetch", "Fetching catalog window...");
// Replace with real HTTP logic.
await Task.Delay(10, cancellationToken);
var metadata = BuildMetadata(builder => builder
.Add("sourceUri", _options!.CatalogUri)
.Add("window", context.Since?.ToString("O") ?? "full"));
yield return CreateRawDocument(
VexDocumentFormat.CsafJson,
new Uri($"{_options.CatalogUri.TrimEnd('/')}/sample.json"),
new byte[] { 0x7B, 0x7D },
metadata);
}
public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
{
var claims = ImmutableArray<VexClaim>.Empty;
var diagnostics = ImmutableDictionary<string, string>.Empty;
return ValueTask.FromResult(new VexClaimBatch(document, claims, diagnostics));
}
}

View File

@@ -0,0 +1,16 @@
using System.ComponentModel.DataAnnotations;
namespace StellaOps.Excititor.Connectors.MyProvider;
public sealed class MyConnectorOptions
{
[Required]
[Url]
public string CatalogUri { get; set; } = default!;
[Required]
public string ApiKey { get; set; } = default!;
[Range(1, 32)]
public int MaxParallelRequests { get; set; } = 4;
}

View File

@@ -0,0 +1,15 @@
using System.Collections.Generic;
using StellaOps.Excititor.Connectors.Abstractions;
namespace StellaOps.Excititor.Connectors.MyProvider;
public sealed class MyConnectorOptionsValidator : IVexConnectorOptionsValidator<MyConnectorOptions>
{
public void Validate(VexConnectorDescriptor descriptor, MyConnectorOptions options, IList<string> errors)
{
if (!options.CatalogUri.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
{
errors.Add("CatalogUri must use HTTPS.");
}
}
}

View File

@@ -0,0 +1,27 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using StellaOps.Plugin;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Connectors.MyProvider;
public sealed class MyConnectorPlugin : IConnectorPlugin
{
private static readonly VexConnectorDescriptor Descriptor = new(
id: "excititor:my-provider",
kind: VexProviderKind.Vendor,
displayName: "My Provider VEX");
public string Name => Descriptor.DisplayName;
public bool IsAvailable(IServiceProvider services) => true;
public IFeedConnector Create(IServiceProvider services)
{
var logger = services.GetRequiredService<ILogger<MyConnector>>();
var timeProvider = services.GetRequiredService<TimeProvider>();
var validators = services.GetServices<IVexConnectorOptionsValidator<MyConnectorOptions>>();
return new MyConnector(Descriptor, logger, timeProvider, validators);
}
}

View File

@@ -1,4 +1,4 @@
# Feedser Apple Security Update Connector Operations
# Concelier Apple Security Update Connector Operations
This runbook covers staging and production rollout for the Apple security updates connector (`source:vndr-apple:*`), including observability checks and fixture maintenance.
@@ -6,10 +6,10 @@ This runbook covers staging and production rollout for the Apple security update
- Network egress (or mirrored cache) for `https://gdmf.apple.com/v2/pmv` and the Apple Support domain (`https://support.apple.com/`).
- Optional: corporate proxy exclusions for the Apple hosts if outbound traffic is normally filtered.
- Updated configuration (environment variables or `feedser.yaml`) with an `apple` section. Example baseline:
- Updated configuration (environment variables or `concelier.yaml`) with an `apple` section. Example baseline:
```yaml
feedser:
concelier:
sources:
apple:
softwareLookupUri: "https://gdmf.apple.com/v2/pmv"
@@ -21,23 +21,23 @@ feedser:
failureBackoff: "00:05:00"
```
> `softwareLookupUri` and `advisoryBaseUri` must stay absolute and aligned with the HTTP allow-list; Feedser automatically adds both hosts to the connector HttpClient.
> `softwareLookupUri` and `advisoryBaseUri` must stay absolute and aligned with the HTTP allow-list; Concelier automatically adds both hosts to the connector HttpClient.
## 2. Staging Smoke Test
1. Deploy the configuration and restart the Feedser workers to ensure the Apple connector options are bound.
1. Deploy the configuration and restart the Concelier workers to ensure the Apple connector options are bound.
2. Trigger a full connector cycle:
- CLI: `stella db jobs run source:vndr-apple:fetch --and-then source:vndr-apple:parse --and-then source:vndr-apple:map`
- REST: `POST /jobs/run { "kind": "source:vndr-apple:fetch", "chain": ["source:vndr-apple:parse", "source:vndr-apple:map"] }`
3. Validate metrics exported under meter `StellaOps.Feedser.Source.Vndr.Apple`:
3. Validate metrics exported under meter `StellaOps.Concelier.Connector.Vndr.Apple`:
- `apple.fetch.items` (documents fetched)
- `apple.fetch.failures`
- `apple.fetch.unchanged`
- `apple.parse.failures`
- `apple.map.affected.count` (histogram of affected package counts)
4. Cross-check the shared HTTP counters:
- `feedser.source.http.requests_total{feedser_source="vndr-apple"}` should increase for both index and detail phases.
- `feedser.source.http.failures_total{feedser_source="vndr-apple"}` should remain flat (0) during a healthy run.
- `concelier.source.http.requests_total{concelier_source="vndr-apple"}` should increase for both index and detail phases.
- `concelier.source.http.failures_total{concelier_source="vndr-apple"}` should remain flat (0) during a healthy run.
5. Inspect the info logs:
- `Apple software index fetch … processed=X newDocuments=Y`
- `Apple advisory parse complete … aliases=… affected=…`
@@ -50,28 +50,28 @@ feedser:
## 3. Production Monitoring
- **Dashboards** Add the following expressions to your Feedser Grafana board (OTLP/Prometheus naming assumed):
- `rate(apple_fetch_items_total[15m])` vs `rate(feedser_source_http_requests_total{feedser_source="vndr-apple"}[15m])`
- **Dashboards** Add the following expressions to your Concelier Grafana board (OTLP/Prometheus naming assumed):
- `rate(apple_fetch_items_total[15m])` vs `rate(concelier_source_http_requests_total{concelier_source="vndr-apple"}[15m])`
- `rate(apple_fetch_failures_total[5m])` for error spikes (`severity=warning` at `>0`)
- `histogram_quantile(0.95, rate(apple_map_affected_count_bucket[1h]))` to watch affected-package fan-out
- `increase(apple_parse_failures_total[6h])` to catch parser drift (alerts at `>0`)
- **Alerts** Page if `rate(apple_fetch_items_total[2h]) == 0` during business hours while other connectors are active. This often indicates lookup feed failures or misconfigured allow-lists.
- **Logs** Surface warnings `Apple document {DocumentId} missing GridFS payload` or `Apple parse failed`—repeated hits imply storage issues or HTML regressions.
- **Telemetry pipeline** `StellaOps.Feedser.WebService` now exports `StellaOps.Feedser.Source.Vndr.Apple` alongside existing Feedser meters; ensure your OTEL collector or Prometheus scraper includes it.
- **Telemetry pipeline** `StellaOps.Concelier.WebService` now exports `StellaOps.Concelier.Connector.Vndr.Apple` alongside existing Concelier meters; ensure your OTEL collector or Prometheus scraper includes it.
## 4. Fixture Maintenance
Regression fixtures live under `src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures`. Refresh them whenever Apple reshapes the HT layout or when new platforms appear.
Regression fixtures live under `src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures`. Refresh them whenever Apple reshapes the HT layout or when new platforms appear.
1. Run the helper script matching your platform:
- Bash: `./scripts/update-apple-fixtures.sh`
- PowerShell: `./scripts/update-apple-fixtures.ps1`
2. Each script exports `UPDATE_APPLE_FIXTURES=1`, updates the `WSLENV` passthrough, and touches `.update-apple-fixtures` so WSL+VS Code test runs observe the flag. The subsequent test execution fetches the live HT articles listed in `AppleFixtureManager`, sanitises the HTML, and rewrites the `.expected.json` DTO snapshots.
3. Review the diff for localisation or nav noise. Once satisfied, re-run the tests without the env var (`dotnet test src/StellaOps.Feedser.Source.Vndr.Apple.Tests/StellaOps.Feedser.Source.Vndr.Apple.Tests.csproj`) to verify determinism.
3. Review the diff for localisation or nav noise. Once satisfied, re-run the tests without the env var (`dotnet test src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/StellaOps.Concelier.Connector.Vndr.Apple.Tests.csproj`) to verify determinism.
4. Commit fixture updates together with any parser/mapping changes that motivated them.
## 5. Known Issues & Follow-up Tasks
- Apple occasionally throttles anonymous requests after bursts. The connector backs off automatically, but persistent `apple.fetch.failures` spikes might require mirroring the HT content or scheduling wider fetch windows.
- Rapid Security Responses may appear before the general patch notes surface in the lookup JSON. When that happens, the fetch run will log `detailFailures>0`. Collect sample HTML and refresh fixtures to confirm parser coverage.
- Multi-locale content is still under regression sweep (`src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md`). Capture non-`en-us` snapshots once the fixture tooling stabilises.
- Multi-locale content is still under regression sweep (`src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md`). Capture non-`en-us` snapshots once the fixture tooling stabilises.

View File

@@ -1,32 +1,32 @@
# Feedser Authority Audit Runbook
# Concelier Authority Audit Runbook
_Last updated: 2025-10-12_
This runbook helps operators verify and monitor the StellaOps Feedser ⇆ Authority integration. It focuses on the `/jobs*` surface, which now requires StellaOps Authority tokens, and the corresponding audit/metric signals that expose authentication and bypass activity.
This runbook helps operators verify and monitor the StellaOps Concelier ⇆ Authority integration. It focuses on the `/jobs*` surface, which now requires StellaOps Authority tokens, and the corresponding audit/metric signals that expose authentication and bypass activity.
## 1. Prerequisites
- Authority integration is enabled in `feedser.yaml` (or via `FEEDSER_AUTHORITY__*` environment variables) with a valid `clientId`, secret, audience, and required scopes.
- OTLP metrics/log exporters are configured (`feedser.telemetry.*`) or container stdout is shipped to your SIEM.
- Operators have access to the Feedser job trigger endpoints via CLI or REST for smoke tests.
- Authority integration is enabled in `concelier.yaml` (or via `CONCELIER_AUTHORITY__*` environment variables) with a valid `clientId`, secret, audience, and required scopes.
- OTLP metrics/log exporters are configured (`concelier.telemetry.*`) or container stdout is shipped to your SIEM.
- Operators have access to the Concelier job trigger endpoints via CLI or REST for smoke tests.
### Configuration snippet
```yaml
feedser:
concelier:
authority:
enabled: true
allowAnonymousFallback: false # keep true only during initial rollout
issuer: "https://authority.internal"
audiences:
- "api://feedser"
- "api://concelier"
requiredScopes:
- "feedser.jobs.trigger"
- "concelier.jobs.trigger"
bypassNetworks:
- "127.0.0.1/32"
- "::1/128"
clientId: "feedser-jobs"
clientSecretFile: "/run/secrets/feedser_authority_client"
clientId: "concelier-jobs"
clientSecretFile: "/run/secrets/concelier_authority_client"
tokenClockSkewSeconds: 60
resilience:
enableRetries: true
@@ -38,22 +38,22 @@ feedser:
offlineCacheTolerance: "00:10:00"
```
> Store secrets outside source control. Feedser reads `clientSecretFile` on startup; rotate by updating the mounted file and restarting the service.
> Store secrets outside source control. Concelier reads `clientSecretFile` on startup; rotate by updating the mounted file and restarting the service.
### Resilience tuning
- **Connected sites:** keep the default 1s / 2s / 5s retry ladder so Feedser retries transient Authority hiccups but still surfaces outages quickly. Leave `allowOfflineCacheFallback=true` so cached discovery/JWKS data can bridge short Pathfinder restarts.
- **Air-gapped/Offline Kit installs:** extend `offlineCacheTolerance` (1530minutes) to keep the cached metadata valid between manual synchronisations. You can also disable retries (`enableRetries=false`) if infrastructure teams prefer to handle exponential backoff at the network layer; Feedser will fail fast but keep deterministic logs.
- Feedser resolves these knobs through `IOptionsMonitor<StellaOpsAuthClientOptions>`. Edits to `feedser.yaml` are applied on configuration reload; restart the container if you change environment variables or do not have file-watch reloads enabled.
- **Connected sites:** keep the default 1s / 2s / 5s retry ladder so Concelier retries transient Authority hiccups but still surfaces outages quickly. Leave `allowOfflineCacheFallback=true` so cached discovery/JWKS data can bridge short Pathfinder restarts.
- **Air-gapped/Offline Kit installs:** extend `offlineCacheTolerance` (1530minutes) to keep the cached metadata valid between manual synchronisations. You can also disable retries (`enableRetries=false`) if infrastructure teams prefer to handle exponential backoff at the network layer; Concelier will fail fast but keep deterministic logs.
- Concelier resolves these knobs through `IOptionsMonitor<StellaOpsAuthClientOptions>`. Edits to `concelier.yaml` are applied on configuration reload; restart the container if you change environment variables or do not have file-watch reloads enabled.
## 2. Key Signals
### 2.1 Audit log channel
Feedser emits structured audit entries via the `Feedser.Authorization.Audit` logger for every `/jobs*` request once Authority enforcement is active.
Concelier emits structured audit entries via the `Concelier.Authorization.Audit` logger for every `/jobs*` request once Authority enforcement is active.
```
Feedser authorization audit route=/jobs/definitions status=200 subject=ops@example.com clientId=feedser-cli scopes=feedser.jobs.trigger bypass=False remote=10.1.4.7
Concelier authorization audit route=/jobs/definitions status=200 subject=ops@example.com clientId=concelier-cli scopes=concelier.jobs.trigger bypass=False remote=10.1.4.7
```
| Field | Sample value | Meaning |
@@ -61,8 +61,8 @@ Feedser authorization audit route=/jobs/definitions status=200 subject=ops@examp
| `route` | `/jobs/definitions` | Endpoint that processed the request. |
| `status` | `200` / `401` / `409` | Final HTTP status code returned to the caller. |
| `subject` | `ops@example.com` | User or service principal subject (falls back to `(anonymous)` when unauthenticated). |
| `clientId` | `feedser-cli` | OAuth client ID provided by Authority ( `(none)` if the token lacked the claim). |
| `scopes` | `feedser.jobs.trigger` | Normalised scope list extracted from token claims; `(none)` if the token carried none. |
| `clientId` | `concelier-cli` | OAuth client ID provided by Authority ( `(none)` if the token lacked the claim). |
| `scopes` | `concelier.jobs.trigger` | Normalised scope list extracted from token claims; `(none)` if the token carried none. |
| `bypass` | `True` / `False` | Indicates whether the request succeeded because its source IP matched a bypass CIDR. |
| `remote` | `10.1.4.7` | Remote IP recorded from the connection / forwarded header test hooks. |
@@ -74,7 +74,7 @@ Use your logging backend (e.g., Loki) to index the logger name and filter for su
### 2.2 Metrics
Feedser publishes counters under the OTEL meter `StellaOps.Feedser.WebService.Jobs`. Tags: `job.kind`, `job.trigger`, `job.outcome`.
Concelier publishes counters under the OTEL meter `StellaOps.Concelier.WebService.Jobs`. Tags: `job.kind`, `job.trigger`, `job.outcome`.
| Metric name | Description | PromQL example |
|-------------------------------|----------------------------------------------------|----------------|
@@ -84,42 +84,42 @@ Feedser publishes counters under the OTEL meter `StellaOps.Feedser.WebService.Jo
> Prometheus/OTEL collectors typically surface counters with `_total` suffix. Adjust queries to match your pipelines generated metric names.
Correlate audit logs with the following global meter exported via `Feedser.SourceDiagnostics`:
Correlate audit logs with the following global meter exported via `Concelier.SourceDiagnostics`:
- `feedser.source.http.requests_total{feedser_source="jobs-run"}` ensures REST/manual triggers route through Authority.
- If Grafana dashboards are deployed, extend the “Feedser Jobs” board with the above counters plus a table of recent audit log entries.
- `concelier.source.http.requests_total{concelier_source="jobs-run"}` ensures REST/manual triggers route through Authority.
- If Grafana dashboards are deployed, extend the “Concelier Jobs” board with the above counters plus a table of recent audit log entries.
## 3. Alerting Guidance
1. **Unauthorized bypass attempt**
- Query: `sum(rate(log_messages_total{logger="Feedser.Authorization.Audit", status="401", bypass="True"}[5m])) > 0`
- Query: `sum(rate(log_messages_total{logger="Concelier.Authorization.Audit", status="401", bypass="True"}[5m])) > 0`
- Action: verify `bypassNetworks` list; confirm expected maintenance windows; rotate credentials if suspicious.
2. **Missing scopes**
- Query: `sum(rate(log_messages_total{logger="Feedser.Authorization.Audit", scopes="(none)", status="200"}[5m])) > 0`
- Action: audit Authority client registration; ensure `requiredScopes` includes `feedser.jobs.trigger`.
- Query: `sum(rate(log_messages_total{logger="Concelier.Authorization.Audit", scopes="(none)", status="200"}[5m])) > 0`
- Action: audit Authority client registration; ensure `requiredScopes` includes `concelier.jobs.trigger`.
3. **Trigger failure surge**
- Query: `sum(rate(web_jobs_trigger_failed_total[10m])) > 0` with severity `warning` if sustained for 10 minutes.
- Action: inspect correlated audit entries and `Feedser.Telemetry` traces for job execution errors.
- Action: inspect correlated audit entries and `Concelier.Telemetry` traces for job execution errors.
4. **Conflict spike**
- Query: `sum(rate(web_jobs_trigger_conflict_total[10m])) > 5` (tune threshold).
- Action: downstream scheduling may be firing repetitive triggers; ensure precedence is configured properly.
5. **Authority offline**
- Watch `Feedser.Authorization.Audit` logs for `status=503` or `status=500` along with `clientId="(none)"`. Investigate Authority availability before re-enabling anonymous fallback.
- Watch `Concelier.Authorization.Audit` logs for `status=503` or `status=500` along with `clientId="(none)"`. Investigate Authority availability before re-enabling anonymous fallback.
## 4. Rollout & Verification Procedure
1. **Pre-checks**
- Confirm `allowAnonymousFallback` is `false` in production; keep `true` only during staged validation.
- Validate Authority issuer metadata is reachable from Feedser (`curl https://authority.internal/.well-known/openid-configuration` from the host).
- Validate Authority issuer metadata is reachable from Concelier (`curl https://authority.internal/.well-known/openid-configuration` from the host).
2. **Smoke test with valid token**
- Obtain a token via CLI: `stella auth login --scope feedser.jobs.trigger`.
- Trigger a read-only endpoint: `curl -H "Authorization: Bearer $TOKEN" https://feedser.internal/jobs/definitions`.
- Expect HTTP 200/202 and an audit log with `bypass=False`, `scopes=feedser.jobs.trigger`.
- Obtain a token via CLI: `stella auth login --scope concelier.jobs.trigger`.
- Trigger a read-only endpoint: `curl -H "Authorization: Bearer $TOKEN" https://concelier.internal/jobs/definitions`.
- Expect HTTP 200/202 and an audit log with `bypass=False`, `scopes=concelier.jobs.trigger`.
3. **Negative test without token**
- Call the same endpoint without a token. Expect HTTP 401, `bypass=False`.
@@ -130,21 +130,21 @@ Correlate audit logs with the following global meter exported via `Feedser.Sourc
5. **Metrics validation**
- Ensure `web.jobs.triggered` counter increments during accepted runs.
- Exporters should show corresponding spans (`feedser.job.trigger`) if tracing is enabled.
- Exporters should show corresponding spans (`concelier.job.trigger`) if tracing is enabled.
## 5. Troubleshooting
| Symptom | Probable cause | Remediation |
|---------|----------------|-------------|
| Audit log shows `clientId=(none)` for all requests | Authority not issuing `client_id` claim or CLI outdated | Update StellaOps Authority configuration (`StellaOpsAuthorityOptions.Token.Claims.ClientId`), or upgrade the CLI token acquisition flow. |
| Requests succeed with `bypass=True` unexpectedly | Local network added to `bypassNetworks` or fallback still enabled | Remove/adjust the CIDR list, disable anonymous fallback, restart Feedser. |
| HTTP 401 with valid token | `requiredScopes` missing from client registration or token audience mismatch | Verify Authority client scopes (`feedser.jobs.trigger`) and ensure the token audience matches `audiences` config. |
| Metrics missing from Prometheus | Telemetry exporters disabled or filter missing OTEL meter | Set `feedser.telemetry.enableMetrics=true`, ensure collector includes `StellaOps.Feedser.WebService.Jobs` meter. |
| Sudden spike in `web.jobs.trigger.failed` | Downstream job failure or Authority timeout mid-request | Inspect Feedser job logs, re-run with tracing enabled, validate Authority latency. |
| Requests succeed with `bypass=True` unexpectedly | Local network added to `bypassNetworks` or fallback still enabled | Remove/adjust the CIDR list, disable anonymous fallback, restart Concelier. |
| HTTP 401 with valid token | `requiredScopes` missing from client registration or token audience mismatch | Verify Authority client scopes (`concelier.jobs.trigger`) and ensure the token audience matches `audiences` config. |
| Metrics missing from Prometheus | Telemetry exporters disabled or filter missing OTEL meter | Set `concelier.telemetry.enableMetrics=true`, ensure collector includes `StellaOps.Concelier.WebService.Jobs` meter. |
| Sudden spike in `web.jobs.trigger.failed` | Downstream job failure or Authority timeout mid-request | Inspect Concelier job logs, re-run with tracing enabled, validate Authority latency. |
## 6. References
- `docs/21_INSTALL_GUIDE.md` Authority configuration quick start.
- `docs/17_SECURITY_HARDENING_GUIDE.md` Security guardrails and enforcement deadlines.
- `docs/ops/authority-monitoring.md` Authority-side monitoring and alerting playbook.
- `StellaOps.Feedser.WebService/Filters/JobAuthorizationAuditFilter.cs` source of audit log fields.
- `StellaOps.Concelier.WebService/Filters/JobAuthorizationAuditFilter.cs` source of audit log fields.

View File

@@ -0,0 +1,72 @@
# Concelier CCCS Connector Operations
This runbook covers daytoday operation of the Canadian Centre for Cyber Security (`source:cccs:*`) connector, including configuration, telemetry, and historical backfill guidance for English/French advisories.
## 1. Configuration Checklist
- Network egress (or mirrored cache) for `https://www.cyber.gc.ca/` and the JSON API endpoints under `/api/cccs/`.
- Set the Concelier options before restarting workers. Example `concelier.yaml` snippet:
```yaml
concelier:
sources:
cccs:
feeds:
- language: "en"
uri: "https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=en&content_type=cccs_threat"
- language: "fr"
uri: "https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=fr&content_type=cccs_threat"
maxEntriesPerFetch: 80 # increase temporarily for backfill runs
maxKnownEntries: 512
requestTimeout: "00:00:30"
requestDelay: "00:00:00.250"
failureBackoff: "00:05:00"
```
> The `/api/cccs/threats/v1/get` endpoint returns thousands of records per language (≈5100 rows each as of 20251014). The connector honours `maxEntriesPerFetch`, so leave it low for steadystate and raise it for planned backfills.
## 2. Telemetry & Logging
- **Metrics (Meter `StellaOps.Concelier.Connector.Cccs`):**
- `cccs.fetch.attempts`, `cccs.fetch.success`, `cccs.fetch.failures`
- `cccs.fetch.documents`, `cccs.fetch.unchanged`
- `cccs.parse.success`, `cccs.parse.failures`, `cccs.parse.quarantine`
- `cccs.map.success`, `cccs.map.failures`
- **Shared HTTP metrics** via `SourceDiagnostics`:
- `concelier.source.http.requests{concelier.source="cccs"}`
- `concelier.source.http.failures{concelier.source="cccs"}`
- `concelier.source.http.duration{concelier.source="cccs"}`
- **Structured logs**
- `CCCS fetch completed feeds=… items=… newDocuments=… pendingDocuments=…`
- `CCCS parse completed parsed=… failures=…`
- `CCCS map completed mapped=… failures=…`
- Warnings fire when GridFS payloads/DTOs go missing or parser sanitisation fails.
Suggested Grafana alerts:
- `increase(cccs.fetch.failures_total[15m]) > 0`
- `rate(cccs.map.success_total[1h]) == 0` while other connectors are active
- `histogram_quantile(0.95, rate(concelier_source_http_duration_bucket{concelier_source="cccs"}[1h])) > 5s`
## 3. Historical Backfill Plan
1. **Snapshot the source** the API accepts `page=<n>` and `lang=<en|fr>` query parameters. `page=0` returns the full dataset (observed earliest `date_created`: 20180608 for EN, 20180608 for FR). Mirror those responses into Offline Kit storage when operating airgapped.
2. **Stage ingestion**:
- Temporarily raise `maxEntriesPerFetch` (e.g. 500) and restart Concelier workers.
- Run chained jobs until `pendingDocuments` drains:
`stella db jobs run source:cccs:fetch --and-then source:cccs:parse --and-then source:cccs:map`
- Monitor `cccs.fetch.unchanged` growth; once it approaches dataset size the backfill is complete.
3. **Optional pagination sweep** for incremental mirrors, iterate `page=<n>` (0…N) while `response.Count == 50`, persisting JSON to disk. Store alongside metadata (`language`, `page`, SHA256) so repeated runs detect drift.
4. **Language split** keep EN/FR payloads separate to preserve canonical language fields. The connector emits `Language` directly from the feed entry, so mixed ingestion simply produces parallel advisories keyed by the same serial number.
5. **Throttle planning** schedule backfills during maintenance windows; the API tolerates burst downloads but respect the 250ms request delay or raise it if mirrored traffic is not available.
## 4. Selector & Sanitiser Notes
- `CccsHtmlParser` now parses the **unsanitised DOM** (via AngleSharp) and only sanitises when persisting `ContentHtml`.
- Product extraction walks headings (`Affected Products`, `Produits touchés`, `Mesures recommandées`) and consumes nested lists within `div/section/article` containers.
- `HtmlContentSanitizer` allows `<h1>…<h6>` and `<section>` so stored HTML keeps headings for UI rendering and downstream summarisation.
## 5. Fixture Maintenance
- Regression fixtures live in `src/StellaOps.Concelier.Connector.Cccs.Tests/Fixtures`.
- Refresh via `UPDATE_CCCS_FIXTURES=1 dotnet test src/StellaOps.Concelier.Connector.Cccs.Tests/StellaOps.Concelier.Connector.Cccs.Tests.csproj`.
- Fixtures capture both EN/FR advisories with nested lists to guard against sanitiser regressions; review diffs for heading/list changes before committing.

View File

@@ -0,0 +1,146 @@
# Concelier CERT-Bund Connector Operations
_Last updated: 2025-10-17_
Germanys Federal Office for Information Security (BSI) operates the Warn- und Informationsdienst (WID) portal. The Concelier CERT-Bund connector (`source:cert-bund:*`) ingests the public RSS feed, hydrates the portals JSON detail endpoint, and maps the result into canonical advisories while preserving the original German content.
---
## 1. Configuration Checklist
- Allow outbound access (or stage mirrors) for:
- `https://wid.cert-bund.de/content/public/securityAdvisory/rss`
- `https://wid.cert-bund.de/portal/` (session/bootstrap)
- `https://wid.cert-bund.de/portal/api/securityadvisory` (detail/search/export JSON)
- Ensure the HTTP client reuses a cookie container (the connectors dependency injection wiring already sets this up).
Example `concelier.yaml` fragment:
```yaml
concelier:
sources:
cert-bund:
feedUri: "https://wid.cert-bund.de/content/public/securityAdvisory/rss"
portalBootstrapUri: "https://wid.cert-bund.de/portal/"
detailApiUri: "https://wid.cert-bund.de/portal/api/securityadvisory"
maxAdvisoriesPerFetch: 50
maxKnownAdvisories: 512
requestTimeout: "00:00:30"
requestDelay: "00:00:00.250"
failureBackoff: "00:05:00"
```
> Leave `maxAdvisoriesPerFetch` at 50 during normal operation. Raise it only for controlled backfills, then restore the default to avoid overwhelming the portal.
---
## 2. Telemetry & Logging
- **Meter**: `StellaOps.Concelier.Connector.CertBund`
- **Counters / histograms**:
- `certbund.feed.fetch.attempts|success|failures`
- `certbund.feed.items.count`
- `certbund.feed.enqueued.count`
- `certbund.feed.coverage.days`
- `certbund.detail.fetch.attempts|success|not_modified|failures{reason}`
- `certbund.parse.success|failures{reason}`
- `certbund.parse.products.count`, `certbund.parse.cve.count`
- `certbund.map.success|failures{reason}`
- `certbund.map.affected.count`, `certbund.map.aliases.count`
- Shared HTTP metrics remain available through `concelier.source.http.*`.
**Structured logs** (all emitted at information level when work occurs):
- `CERT-Bund fetch cycle: … truncated {Truncated}, coverageDays={CoverageDays}`
- `CERT-Bund parse cycle: parsed {Parsed}, failures {Failures}, …`
- `CERT-Bund map cycle: mapped {Mapped}, failures {Failures}, …`
Alerting ideas:
1. `increase(certbund.detail.fetch.failures_total[10m]) > 0`
2. `rate(certbund.map.success_total[30m]) == 0`
3. `histogram_quantile(0.95, rate(concelier_source_http_duration_bucket{concelier_source="cert-bund"}[15m])) > 5s`
The WebService now registers the meter so metrics surface automatically once OpenTelemetry metrics are enabled.
---
## 3. Historical Backfill & Export Strategy
### 3.1 Retention snapshot
- RSS window: ~250 advisories (≈90days at current cadence).
- Older advisories are accessible through the JSON search/export APIs once the anti-CSRF token is supplied.
### 3.2 JSON search pagination
```bash
# 1. Bootstrap cookies (client_config + XSRF-TOKEN)
curl -s -c cookies.txt "https://wid.cert-bund.de/portal/" > /dev/null
curl -s -b cookies.txt -c cookies.txt \
-H "X-Requested-With: XMLHttpRequest" \
"https://wid.cert-bund.de/portal/api/security/csrf" > /dev/null
XSRF=$(awk '/XSRF-TOKEN/ {print $7}' cookies.txt)
# 2. Page search results
curl -s -b cookies.txt \
-H "Content-Type: application/json" \
-H "Accept: application/json" \
-H "X-XSRF-TOKEN: ${XSRF}" \
-X POST \
--data '{"page":4,"size":100,"sort":["published,desc"]}' \
"https://wid.cert-bund.de/portal/api/securityadvisory/search" \
> certbund-page4.json
```
Iterate `page` until the response `content` array is empty. Pages 09 currently cover 2014→present. Persist JSON responses (plus SHA256) for Offline Kit parity.
> **Shortcut** run `python tools/certbund_offline_snapshot.py --output seed-data/cert-bund`
> to bootstrap the session, capture the paginated search responses, and regenerate
> the manifest/checksum files automatically. Supply `--cookie-file` and `--xsrf-token`
> if the portal requires a browser-derived session (see options via `--help`).
### 3.3 Export bundles
```bash
python tools/certbund_offline_snapshot.py \
--output seed-data/cert-bund \
--start-year 2014 \
--end-year "$(date -u +%Y)"
```
The helper stores yearly exports under `seed-data/cert-bund/export/`,
captures paginated search snapshots in `seed-data/cert-bund/search/`,
and generates the manifest + SHA files in `seed-data/cert-bund/manifest/`.
Split ranges according to your compliance window (default: one file per
calendar year). Concelier can ingest these JSON payloads directly when
operating offline.
> When automatic bootstrap fails (e.g. portal introduces CAPTCHA), run the
> manual `curl` flow above, then rerun the helper with `--skip-fetch` to
> rebuild the manifest from the existing files.
### 3.4 Connector-driven catch-up
1. Temporarily raise `maxAdvisoriesPerFetch` (e.g. 150) and reduce `requestDelay`.
2. Run `stella db jobs run source:cert-bund:fetch --and-then source:cert-bund:parse --and-then source:cert-bund:map` until the fetch log reports `enqueued=0`.
3. Restore defaults and capture the cursor snapshot for audit.
---
## 4. Locale & Translation Guidance
- Advisories remain in German (`language: "de"`). Preserve wording for provenance and legal accuracy.
- UI localisation: enable the translation bundles documented in `docs/15_UI_GUIDE.md` if English UI copy is required. Operators can overlay machine or human translations, but the canonical database stores the source text.
- Docs guild is compiling a CERT-Bund terminology glossary under `docs/locale/certbund-glossary.md` so downstream teams can reference consistent English equivalents without altering the stored advisories.
---
## 5. Verification Checklist
1. Observe `certbund.feed.fetch.success` and `certbund.detail.fetch.success` increments after runs; `certbund.feed.coverage.days` should hover near the observed RSS window.
2. Ensure summary logs report `truncated=false` in steady state—`true` indicates the fetch cap was hit.
3. During backfills, watch `certbund.feed.enqueued.count` trend to zero.
4. Spot-check stored advisories in Mongo to confirm `language="de"` and reference URLs match the portal detail endpoint.
5. For Offline Kit exports, validate SHA256 hashes before distribution.

View File

@@ -0,0 +1,94 @@
# Concelier Cisco PSIRT Connector OAuth Provisioning SOP
_Last updated: 2025-10-14_
## 1. Scope
This runbook describes how Ops provisions, rotates, and distributes Cisco PSIRT openVuln OAuth client credentials for the Concelier Cisco connector. It covers online and air-gapped (Offline Kit) environments, quota-aware execution, and escalation paths.
## 2. Prerequisites
- Active Cisco.com (CCO) account with access to the Cisco API Console.
- Cisco PSIRT openVuln API entitlement (visible under “My Apps & Keys” once granted).citeturn3search0
- Concelier configuration location (typically `/etc/stella/concelier.yaml` in production) or Offline Kit secret bundle staging directory.
## 3. Provisioning workflow
1. **Register the application**
- Sign in at <https://apiconsole.cisco.com>.
- Select **Register a New App** → Application Type: `Service`, Grant Type: `Client Credentials`, API: `Cisco PSIRT openVuln API`.citeturn3search0
- Record the generated `clientId` and `clientSecret` in the Ops vault.
2. **Verify token issuance**
- Request an access token with:
```bash
curl -s https://id.cisco.com/oauth2/default/v1/token \
-H "Content-Type: application/x-www-form-urlencoded" \
-d "grant_type=client_credentials" \
-d "client_id=${CLIENT_ID}" \
-d "client_secret=${CLIENT_SECRET}"
```
- Confirm HTTP 200 and an `expires_in` value of 3600 seconds (tokens live for one hour).citeturn3search0turn3search7
- Preserve the response only long enough to validate syntax; do **not** persist tokens.
3. **Authorize Concelier runtime**
- Update `concelier:sources:cisco:auth` (or the module-specific secret template) with the stored credentials.
- For Offline Kit delivery, export encrypted secrets into `offline-kit/secrets/cisco-openvuln.json` using the platforms sealed secret format.
4. **Connectivity validation**
- From the Concelier control plane, run `stella db jobs run source:vndr-cisco:fetch --dry-run`.
- Ensure the Source HTTP diagnostics record `Bearer` authorization headers and no 401/403 responses.
## 4. Rotation SOP
| Step | Owner | Notes |
| --- | --- | --- |
| 1. Schedule rotation | Ops (monthly board) | Rotate every 90 days or immediately after suspected credential exposure. |
| 2. Create replacement app | Ops | Repeat §3.1 with “-next” suffix; verify token issuance. |
| 3. Stage dual credentials | Ops + Concelier On-Call | Publish new credentials to secret store alongside current pair. |
| 4. Cut over | Concelier On-Call | Restart connector workers during a low-traffic window (<10 min) to pick up the new secret. |
| 5. Deactivate legacy app | Ops | Delete prior app in Cisco API Console once telemetry confirms successful fetch/parse cycles for 2 consecutive hours. |
**Automation hooks**
- Rotation reminders are tracked in OpsRunbookOps board (`OPS-RUN-KEYS` swim lane); add checklist items for Concelier Cisco when opening a rotation task.
- Use the secret management pipeline (`ops/secrets/rotate.sh --connector cisco`) to template vault updates; the script renders a redacted diff for audit.
## 5. Offline Kit packaging
1. Generate the credential bundle using the Offline Kit CLI:
`offline-kit secrets add cisco-openvuln --client-id … --client-secret …`
2. Store the encrypted payload under `offline-kit/secrets/cisco-openvuln.enc`.
3. Distribute via the Offline Kit channel; update `offline-kit/MANIFEST.md` with the credential fingerprint (SHA256 of plaintext concatenated with metadata).
4. Document validation steps for the receiving site (token request from an air-gapped relay or cached token mirror).
## 6. Quota and throttling guidance
- Cisco enforces combined limits of 5 requests/second, 30 requests/minute, and 5000 requests/day per application.citeturn0search0turn3search6
- Concelier fetch jobs must respect `Retry-After` headers on HTTP 429 responses; Ops should monitor for sustained quota saturation and consider paging window adjustments.
- Telemetry to watch: `concelier.source.http.requests{concelier.source="vndr-cisco"}`, `concelier.source.http.failures{...}`, and connector-specific metrics once implemented.
## 7. Telemetry & Monitoring
- **Metrics (Meter `StellaOps.Concelier.Connector.Vndr.Cisco`)**
- `cisco.fetch.documents`, `cisco.fetch.failures`, `cisco.fetch.unchanged`
- `cisco.parse.success`, `cisco.parse.failures`
- `cisco.map.success`, `cisco.map.failures`, `cisco.map.affected.packages`
- **Shared HTTP metrics** via `SourceDiagnostics`:
- `concelier.source.http.requests{concelier.source="vndr-cisco"}`
- `concelier.source.http.failures{concelier.source="vndr-cisco"}`
- `concelier.source.http.duration{concelier.source="vndr-cisco"}`
- **Structured logs**
- `Cisco fetch completed date=… pages=… added=…` (info)
- `Cisco parse completed parsed=… failures=…` (info)
- `Cisco map completed mapped=… failures=…` (info)
- Warnings surface when DTO serialization fails or GridFS payload is missing.
- Suggested alerts: non-zero `cisco.fetch.failures` in 15m, or `cisco.map.success` flatlines while fetch continues.
## 8. Incident response
- **Token compromise** revoke the application in the Cisco API Console, purge cached secrets, rotate immediately per §4.
- **Persistent 401/403** confirm credentials in vault, then validate token issuance; if unresolved, open a Cisco DevNet support ticket referencing the application ID.
- **429 spikes** inspect job scheduler cadence and adjust connector options (`maxRequestsPerWindow`) before requesting higher quotas from Cisco.
## 9. References
- Cisco PSIRT openVuln API Authentication Guide.citeturn3search0
- Accessing the openVuln API using curl (token lifetime).citeturn3search7
- openVuln API rate limit documentation.citeturn0search0turn3search6

View File

@@ -0,0 +1,160 @@
# Concelier Conflict Resolution Runbook (Sprint 3)
This runbook equips Concelier operators to detect, triage, and resolve advisory conflicts now that the Sprint 3 merge engine landed (`AdvisoryPrecedenceMerger`, merge-event hashing, and telemetry counters). It builds on the canonical rules defined in `src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md` and the metrics/logging instrumentation delivered this sprint.
---
## 1. Precedence Model (recap)
- **Default ranking:** `GHSA -> NVD -> OSV`, with distro/vendor PSIRTs outranking ecosystem feeds (`AdvisoryPrecedenceDefaults`). Use `concelier:merge:precedence:ranks` to override per source when incident response requires it.
- **Freshness override:** if a lower-ranked source is >= 48 hours newer for a freshness-sensitive field (title, summary, affected ranges, references, credits), it wins. Every override stamps `provenance[].decisionReason = freshness`.
- **Tie-breakers:** when precedence and freshness tie, the engine falls back to (1) primary source order, (2) shortest normalized text, (3) lowest stable hash. Merge-generated provenance records set `decisionReason = tie-breaker`.
- **Audit trail:** each merged advisory receives a `merge` provenance entry listing the participating sources plus a `merge_event` record with canonical before/after SHA-256 hashes.
---
## 2. Telemetry Shipped This Sprint
| Instrument | Type | Key Tags | Purpose |
|------------|------|----------|---------|
| `concelier.merge.operations` | Counter | `inputs` | Total precedence merges executed. |
| `concelier.merge.overrides` | Counter | `primary_source`, `suppressed_source`, `primary_rank`, `suppressed_rank` | Field-level overrides chosen by precedence. |
| `concelier.merge.range_overrides` | Counter | `advisory_key`, `package_type`, `primary_source`, `suppressed_source`, `primary_range_count`, `suppressed_range_count` | Package range overrides emitted by `AffectedPackagePrecedenceResolver`. |
| `concelier.merge.conflicts` | Counter | `type` (`severity`, `precedence_tie`), `reason` (`mismatch`, `primary_missing`, `equal_rank`) | Conflicts requiring operator review. |
| `concelier.merge.identity_conflicts` | Counter | `scheme`, `alias_value`, `advisory_count` | Alias collisions surfaced by the identity graph. |
### Structured logs
- `AdvisoryOverride` (EventId 1000) - logs merge suppressions with alias/provenance counts.
- `PackageRangeOverride` (EventId 1001) - logs package-level precedence decisions.
- `PrecedenceConflict` (EventId 1002) - logs mismatched severity or equal-rank scenarios.
- `Alias collision ...` (no EventId) - emitted when `concelier.merge.identity_conflicts` increments.
Expect all logs at `Information`. Ensure OTEL exporters include the scope `StellaOps.Concelier.Merge`.
---
## 3. Detection & Alerting
1. **Dashboard panels**
- `concelier.merge.conflicts` - table grouped by `type/reason`. Alert when > 0 in a 15 minute window.
- `concelier.merge.range_overrides` - stacked bar by `package_type`. Spikes highlight vendor PSIRT overrides over registry data.
- `concelier.merge.overrides` with `primary_source|suppressed_source` - catches unexpected precedence flips (e.g., OSV overtaking GHSA).
- `concelier.merge.identity_conflicts` - single-stat; alert when alias collisions occur more than once per day.
2. **Log based alerts**
- `eventId=1002` with `reason="equal_rank"` - indicates precedence table gaps; page merge owners.
- `eventId=1002` with `reason="mismatch"` - severity disagreement; open connector bug if sustained.
3. **Job health**
- `stellaops-cli db merge` exit code `1` signifies unresolved conflicts. Pipe to automation that captures logs and notifies #concelier-ops.
### Threshold updates (2025-10-12)
- `concelier.merge.conflicts` Page only when ≥ 2 events fire within 30 minutes; the synthetic conflict fixture run produces 0 conflicts, so the first event now routes to Slack for manual review instead of paging.
- `concelier.merge.overrides` Raise a warning when the 30-minute sum exceeds 10 (canonical triple yields exactly 1 summary override with `primary_source=osv`, `suppressed_source=ghsa`).
- `concelier.merge.range_overrides` Maintain the 15-minute alert at ≥ 3 but annotate dashboards that the regression triple emits a single `package_type=semver` override so ops can spot unexpected spikes.
---
## 4. Triage Workflow
1. **Confirm job context**
- `stellaops-cli db merge` (CLI) or `POST /jobs/merge:reconcile` (API) to rehydrate the merge job. Use `--verbose` to stream structured logs during triage.
2. **Inspect metrics**
- Correlate spikes in `concelier.merge.conflicts` with `primary_source`/`suppressed_source` tags from `concelier.merge.overrides`.
3. **Pull structured logs**
- Example (vector output):
```
jq 'select(.EventId.Name=="PrecedenceConflict") | {advisory: .State[0].Value, type: .ConflictType, reason: .Reason, primary: .PrimarySources, suppressed: .SuppressedSources}' stellaops-concelier.log
```
4. **Review merge events**
- `mongosh`:
```javascript
use concelier;
db.merge_event.find({ advisoryKey: "CVE-2025-1234" }).sort({ mergedAt: -1 }).limit(5);
```
- Compare `beforeHash` vs `afterHash` to confirm the merge actually changed canonical output.
5. **Interrogate provenance**
- `db.advisories.findOne({ advisoryKey: "CVE-2025-1234" }, { title: 1, severity: 1, provenance: 1, "affectedPackages.provenance": 1 })`
- Check `provenance[].decisionReason` values (`precedence`, `freshness`, `tie-breaker`) to understand why the winning field was chosen.
---
## 5. Conflict Classification Matrix
| Signal | Likely Cause | Immediate Action |
|--------|--------------|------------------|
| `reason="mismatch"` with `type="severity"` | Upstream feeds disagree on CVSS vector/severity. | Verify which feed is freshest; if correctness is known, adjust connector mapping or precedence override. |
| `reason="primary_missing"` | Higher-ranked source lacks the field entirely. | Backfill connector data or temporarily allow lower-ranked source via precedence override. |
| `reason="equal_rank"` | Two feeds share the same precedence rank (custom config or missing entry). | Update `concelier:merge:precedence:ranks` to break the tie; restart merge job. |
| Rising `concelier.merge.range_overrides` for a package type | Vendor PSIRT now supplies richer ranges. | Validate connectors emit `decisionReason="precedence"` and update dashboards to treat registry ranges as fallback. |
| `concelier.merge.identity_conflicts` > 0 | Alias scheme mapping produced collisions (duplicate CVE <-> advisory pairs). | Inspect `Alias collision` log payload; reconcile the alias graph by adjusting connector alias output. |
---
## 6. Resolution Playbook
1. **Connector data fix**
- Re-run the offending connector stages (`stellaops-cli db fetch --source ghsa --stage map` etc.).
- Once fixed, rerun merge and verify `decisionReason` reflects `freshness` or `precedence` as expected.
2. **Temporary precedence override**
- Edit `etc/concelier.yaml`:
```yaml
concelier:
merge:
precedence:
ranks:
osv: 1
ghsa: 0
```
- Restart Concelier workers; confirm tags in `concelier.merge.overrides` show the new ranks.
- Document the override with expiry in the change log.
3. **Alias remediation**
- Update connector mapping rules to weed out duplicate aliases (e.g., skip GHSA aliases that mirror CVE IDs).
- Flush cached alias graphs if necessary (`db.alias_graph.drop()` is destructive-coordinate with Storage before issuing).
4. **Escalation**
- If override metrics spike due to upstream regression, open an incident with Security Guild, referencing merge logs and `merge_event` IDs.
---
## 7. Validation Checklist
- [ ] Merge job rerun returns exit code `0`.
- [ ] `concelier.merge.conflicts` baseline returns to zero after corrective action.
- [ ] Latest `merge_event` entry shows expected hash delta.
- [ ] Affected advisory document shows updated `provenance[].decisionReason`.
- [ ] Ops change log updated with incident summary, config overrides, and rollback plan.
---
## 8. Reference Material
- Canonical conflict rules: `src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md`.
- Merge engine internals: `src/StellaOps.Concelier.Merge/Services/AdvisoryPrecedenceMerger.cs`.
- Metrics definitions: `src/StellaOps.Concelier.Merge/Services/AdvisoryMergeService.cs` (identity conflicts) and `AdvisoryPrecedenceMerger`.
- Storage audit trail: `src/StellaOps.Concelier.Merge/Services/MergeEventWriter.cs`, `src/StellaOps.Concelier.Storage.Mongo/MergeEvents`.
Keep this runbook synchronized with future sprint notes and update alert thresholds as baseline volumes change.
---
## 9. Synthetic Regression Fixtures
- **Locations** Canonical conflict snapshots now live at `src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/conflict-ghsa.canonical.json`, `src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/conflict-nvd.canonical.json`, and `src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/conflict-osv.canonical.json`.
- **Validation commands** To regenerate and verify the fixtures offline, run:
```bash
dotnet test src/StellaOps.Concelier.Connector.Ghsa.Tests/StellaOps.Concelier.Connector.Ghsa.Tests.csproj --filter GhsaConflictFixtureTests
dotnet test src/StellaOps.Concelier.Connector.Nvd.Tests/StellaOps.Concelier.Connector.Nvd.Tests.csproj --filter NvdConflictFixtureTests
dotnet test src/StellaOps.Concelier.Connector.Osv.Tests/StellaOps.Concelier.Connector.Osv.Tests.csproj --filter OsvConflictFixtureTests
dotnet test src/StellaOps.Concelier.Merge.Tests/StellaOps.Concelier.Merge.Tests.csproj --filter MergeAsync_AppliesCanonicalRulesAndPersistsDecisions
```
- **Expected signals** The triple produces one freshness-driven summary override (`primary_source=osv`, `suppressed_source=ghsa`) and one range override for the npm SemVer package while leaving `concelier.merge.conflicts` at zero. Use these values as the baseline when tuning dashboards or load-testing alert pipelines.
---
## 10. Change Log
| Date (UTC) | Change | Notes |
|------------|--------|-------|
| 2025-10-16 | Ops review signed off after connector expansion (CCCS, CERT-Bund, KISA, ICS CISA, MSRC) landed. Alert thresholds from §3 reaffirmed; dashboards updated to watch attachment signals emitted by ICS CISA connector. | Ops sign-off recorded by Concelier Ops Guild; no additional overrides required. |

View File

@@ -1,6 +1,6 @@
{
"title": "Feedser CVE & KEV Observability",
"uid": "feedser-cve-kev",
"title": "Concelier CVE & KEV Observability",
"uid": "concelier-cve-kev",
"schemaVersion": 38,
"version": 1,
"editable": true,

View File

@@ -1,4 +1,4 @@
# Feedser CVE & KEV Connector Operations
# Concelier CVE & KEV Connector Operations
This playbook equips operators with the steps required to roll out and monitor the CVE Services and CISA KEV connectors across environments.
@@ -7,17 +7,18 @@ This playbook equips operators with the steps required to roll out and monitor t
### 1.1 Prerequisites
- CVE Services API credentials (organisation ID, user ID, API key) with access to the JSON 5 API.
- Network egress to `https://cveawg.mitre.org` (or a mirrored endpoint) from the Feedser workers.
- Updated `feedser.yaml` (or the matching environment variables) with the following section:
- Network egress to `https://cveawg.mitre.org` (or a mirrored endpoint) from the Concelier workers.
- Updated `concelier.yaml` (or the matching environment variables) with the following section:
```yaml
feedser:
concelier:
sources:
cve:
baseEndpoint: "https://cveawg.mitre.org/api/"
apiOrg: "ORG123"
apiUser: "user@example.org"
apiKeyFile: "/var/run/secrets/feedser/cve-api-key"
apiKeyFile: "/var/run/secrets/concelier/cve-api-key"
seedDirectory: "./seed-data/cve"
pageSize: 200
maxPagesPerFetch: 5
initialBackfill: "30.00:00:00"
@@ -25,31 +26,53 @@ feedser:
failureBackoff: "00:10:00"
```
> Store the API key outside source control. When using `apiKeyFile`, mount the secret file into the container/host; alternatively supply `apiKey` via `FEEDSER_SOURCES__CVE__APIKEY`.
> Store the API key outside source control. When using `apiKeyFile`, mount the secret file into the container/host; alternatively supply `apiKey` via `CONCELIER_SOURCES__CVE__APIKEY`.
> 🪙 When credentials are not yet available, configure `seedDirectory` to point at mirrored CVE JSON (for example, the repos `seed-data/cve/` bundle). The connector will ingest those records and log a warning instead of failing the job; live fetching resumes automatically once `apiOrg` / `apiUser` / `apiKey` are supplied.
### 1.2 Smoke Test (staging)
1. Deploy the updated configuration and restart the Feedser service so the connector picks up the credentials.
1. Deploy the updated configuration and restart the Concelier service so the connector picks up the credentials.
2. Trigger one end-to-end cycle:
- Feedser CLI: `stella db jobs run source:cve:fetch --and-then source:cve:parse --and-then source:cve:map`
- Concelier CLI: `stella db jobs run source:cve:fetch --and-then source:cve:parse --and-then source:cve:map`
- REST fallback: `POST /jobs/run { "kind": "source:cve:fetch", "chain": ["source:cve:parse", "source:cve:map"] }`
3. Observe the following metrics (exported via OTEL meter `StellaOps.Feedser.Source.Cve`):
3. Observe the following metrics (exported via OTEL meter `StellaOps.Concelier.Connector.Cve`):
- `cve.fetch.attempts`, `cve.fetch.success`, `cve.fetch.documents`, `cve.fetch.failures`, `cve.fetch.unchanged`
- `cve.parse.success`, `cve.parse.failures`, `cve.parse.quarantine`
- `cve.map.success`
4. Verify Prometheus shows matching `feedser.source.http.requests_total{feedser_source="cve"}` deltas (list vs detail phases) while `feedser.source.http.failures_total{feedser_source="cve"}` stays flat.
4. Verify Prometheus shows matching `concelier.source.http.requests_total{concelier_source="cve"}` deltas (list vs detail phases) while `concelier.source.http.failures_total{concelier_source="cve"}` stays flat.
5. Confirm the info-level summary log `CVEs fetch window … pages=X detailDocuments=Y detailFailures=Z` appears once per fetch run and shows `detailFailures=0`.
6. Verify the MongoDB advisory store contains fresh CVE advisories (`advisoryKey` prefix `cve/`) and that the source cursor (`source_states` collection) advanced.
### 1.3 Production Monitoring
- **Dashboards** Plot `rate(cve_fetch_success_total[5m])`, `rate(cve_fetch_failures_total[5m])`, and `rate(cve_fetch_documents_total[5m])` alongside `feedser_source_http_requests_total{feedser_source="cve"}` to confirm HTTP and connector counters stay aligned. Keep `feedser.range.primitives{scheme=~"semver|vendor"}` on the same board for range coverage. Example alerts:
- **Dashboards** Plot `rate(cve_fetch_success_total[5m])`, `rate(cve_fetch_failures_total[5m])`, and `rate(cve_fetch_documents_total[5m])` alongside `concelier_source_http_requests_total{concelier_source="cve"}` to confirm HTTP and connector counters stay aligned. Keep `concelier.range.primitives{scheme=~"semver|vendor"}` on the same board for range coverage. Example alerts:
- `rate(cve_fetch_failures_total[5m]) > 0` for 10minutes (`severity=warning`)
- `rate(cve_map_success_total[15m]) == 0` while `rate(cve_fetch_success_total[15m]) > 0` (`severity=critical`)
- `sum_over_time(cve_parse_quarantine_total[1h]) > 0` to catch schema anomalies
- **Logs** Monitor warnings such as `Failed fetching CVE record {CveId}` and `Malformed CVE JSON`, and surface the summary info log `CVEs fetch window … detailFailures=0 detailUnchanged=0` on dashboards. A non-zero `detailFailures` usually indicates rate-limit or auth issues on detail requests.
- **Grafana pack** Import `docs/ops/feedser-cve-kev-grafana-dashboard.json` and filter by panel legend (`CVE`, `KEV`) to reuse the canned layout.
- **Backfill window** Operators can tighten or widen `initialBackfill` / `maxPagesPerFetch` after validating throughput. Update config and restart Feedser to apply changes.
- **Grafana pack** Import `docs/ops/concelier-cve-kev-grafana-dashboard.json` and filter by panel legend (`CVE`, `KEV`) to reuse the canned layout.
- **Backfill window** Operators can tighten or widen `initialBackfill` / `maxPagesPerFetch` after validating throughput. Update config and restart Concelier to apply changes.
### 1.4 Staging smoke log (2025-10-15)
While Ops finalises long-lived CVE Services credentials, we validated the connector end-to-end against the recorded CVE-2024-0001 payloads used in regression tests:
- Command: `dotnet test src/StellaOps.Concelier.Connector.Cve.Tests/StellaOps.Concelier.Connector.Cve.Tests.csproj -l "console;verbosity=detailed"`
- Summary log emitted by the connector:
```
CVEs fetch window 2024-09-01T00:00:00Z->2024-10-01T00:00:00Z pages=1 listSuccess=1 detailDocuments=1 detailFailures=0 detailUnchanged=0 pendingDocuments=0->1 pendingMappings=0->1 hasMorePages=False nextWindowStart=2024-09-15T12:00:00Z nextWindowEnd=(none) nextPage=1
```
- Telemetry captured by `Meter` `StellaOps.Concelier.Connector.Cve`:
| Metric | Value |
|--------|-------|
| `cve.fetch.attempts` | 1 |
| `cve.fetch.success` | 1 |
| `cve.fetch.documents` | 1 |
| `cve.parse.success` | 1 |
| `cve.map.success` | 1 |
The Grafana pack `docs/ops/concelier-cve-kev-grafana-dashboard.json` has been imported into staging so the panels referenced above render against these counters once the live API keys are in place.
## 2. CISA KEV Connector (`source:kev:*`)
@@ -57,10 +80,10 @@ feedser:
- Network egress (or mirrored content) for `https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json`.
- No credentials are required, but the HTTP allow-list must include `www.cisa.gov`.
- Confirm the following snippet in `feedser.yaml` (defaults shown; tune as needed):
- Confirm the following snippet in `concelier.yaml` (defaults shown; tune as needed):
```yaml
feedser:
concelier:
sources:
kev:
feedUri: "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"
@@ -82,15 +105,15 @@ Treat repeated schema failures or growing anomaly counts as an upstream regressi
### 2.3 Smoke Test (staging)
1. Deploy the configuration and restart Feedser.
1. Deploy the configuration and restart Concelier.
2. Trigger a pipeline run:
- CLI: `stella db jobs run source:kev:fetch --and-then source:kev:parse --and-then source:kev:map`
- REST: `POST /jobs/run { "kind": "source:kev:fetch", "chain": ["source:kev:parse", "source:kev:map"] }`
3. Verify the metrics exposed by meter `StellaOps.Feedser.Source.Kev`:
3. Verify the metrics exposed by meter `StellaOps.Concelier.Connector.Kev`:
- `kev.fetch.attempts`, `kev.fetch.success`, `kev.fetch.unchanged`, `kev.fetch.failures`
- `kev.parse.entries` (tag `catalogVersion`), `kev.parse.failures`, `kev.parse.anomalies` (tag `reason`)
- `kev.map.advisories` (tag `catalogVersion`)
4. Confirm `feedser.source.http.requests_total{feedser_source="kev"}` increments once per fetch and that the paired `feedser.source.http.failures_total` stays flat (zero increase).
4. Confirm `concelier.source.http.requests_total{concelier_source="kev"}` increments once per fetch and that the paired `concelier.source.http.failures_total` stays flat (zero increase).
5. Inspect the info logs `Fetched KEV catalog document … pendingDocuments=…` and `Parsed KEV catalog document … entries=…`—they should appear exactly once per run and `Mapped X/Y… skipped=0` should match the `kev.map.advisories` delta.
6. Confirm MongoDB documents exist for the catalog JSON (`raw_documents` & `dtos`) and that advisories with prefix `kev/` are written.
@@ -103,7 +126,7 @@ Treat repeated schema failures or growing anomaly counts as an upstream regressi
### 2.5 Known good dashboard tiles
Add the following panels to the Feedser observability board:
Add the following panels to the Concelier observability board:
| Metric | Recommended visualisation |
|--------|---------------------------|
@@ -117,4 +140,4 @@ Add the following panels to the Feedser observability board:
- Record staging/production smoke test results (date, catalog version, advisory counts) in your teams change log.
- Add the CVE/KEV job kinds to the standard maintenance checklist so operators can manually trigger them after planned downtime.
- Keep this document in sync with future connector changes (for example, new anomaly reasons or additional metrics).
- Version-control dashboard tweaks alongside `docs/ops/feedser-cve-kev-grafana-dashboard.json` so operations can re-import the observability pack during restores.
- Version-control dashboard tweaks alongside `docs/ops/concelier-cve-kev-grafana-dashboard.json` so operations can re-import the observability pack during restores.

View File

@@ -1,6 +1,6 @@
# Feedser GHSA Connector Operations Runbook
# Concelier GHSA Connector Operations Runbook
_Last updated: 2025-10-12_
_Last updated: 2025-10-16_
## 1. Overview
The GitHub Security Advisories (GHSA) connector pulls advisory metadata from the GitHub REST API `/security/advisories` endpoint. GitHub enforces both primary and secondary rate limits, so operators must monitor usage and configure retries to avoid throttling incidents.
@@ -32,9 +32,9 @@ When GitHub reports zero remaining calls, the connector logs and sleeps for the
After the quota recovers above the warning threshold the connector writes an informational log with the refreshed remaining/headroom, letting operators clear alerts quickly.
## 4. Configuration knobs (`feedser.yaml`)
## 4. Configuration knobs (`concelier.yaml`)
```yaml
feedser:
concelier:
sources:
ghsa:
apiToken: "${GITHUB_PAT}"
@@ -58,18 +58,18 @@ feedser:
| `source:ghsa:parse` | `3,13,23,33,43,53 * * * *` | 5 minutes | 4 minutes |
| `source:ghsa:map` | `5,15,25,35,45,55 * * * *` | 5 minutes | 4 minutes |
These defaults spread GHSA stages across the hour so fetch completes before parse/map fire. Override them via `feedser.jobs.definitions[...]` when coordinating multiple connectors on the same runner.
These defaults spread GHSA stages across the hour so fetch completes before parse/map fire. Override them via `concelier.jobs.definitions[...]` when coordinating multiple connectors on the same runner.
## 5. Provisioning credentials
Feedser requires a GitHub personal access token (classic) with the **`read:org`** and **`security_events`** scopes to pull GHSA data. Store it as a secret and reference it via `feedser.sources.ghsa.apiToken`.
Concelier requires a GitHub personal access token (classic) with the **`read:org`** and **`security_events`** scopes to pull GHSA data. Store it as a secret and reference it via `concelier.sources.ghsa.apiToken`.
### Docker Compose (stack operators)
```yaml
services:
feedser:
concelier:
environment:
FEEDSER__SOURCES__GHSA__APITOKEN: /run/secrets/ghsa_pat
CONCELIER__SOURCES__GHSA__APITOKEN: /run/secrets/ghsa_pat
secrets:
- ghsa_pat
@@ -80,25 +80,25 @@ secrets:
### Helm values (cluster operators)
```yaml
feedser:
concelier:
extraEnv:
- name: FEEDSER__SOURCES__GHSA__APITOKEN
- name: CONCELIER__SOURCES__GHSA__APITOKEN
valueFrom:
secretKeyRef:
name: feedser-ghsa
name: concelier-ghsa
key: apiToken
extraSecrets:
feedser-ghsa:
concelier-ghsa:
apiToken: "<paste PAT here or source from external secret store>"
```
After rotating the PAT, restart the Feedser workers (or run `kubectl rollout restart deployment/feedser`) to ensure the configuration reloads.
After rotating the PAT, restart the Concelier workers (or run `kubectl rollout restart deployment/concelier`) to ensure the configuration reloads.
When enabling GHSA the first time, run a staged backfill:
1. Trigger `source:ghsa:fetch` manually (CLI or API) outside of peak hours.
2. Watch `feedser.jobs.health` for the GHSA jobs until they report `healthy`.
2. Watch `concelier.jobs.health` for the GHSA jobs until they report `healthy`.
3. Allow the scheduled cron cadence to resume once the initial backlog drains (typically < 30 minutes).
## 6. Runbook steps when throttled
@@ -107,10 +107,17 @@ When enabling GHSA the first time, run a staged backfill:
3. If rate limits stay exhausted:
- Verify no other jobs are sharing the PAT.
- Temporarily reduce `MaxPagesPerFetch` or `PageSize` to shrink burst size.
- Consider provisioning a dedicated PAT (GHSA permissions only) for Feedser.
- Consider provisioning a dedicated PAT (GHSA permissions only) for Concelier.
4. After the quota resets, reset `rateLimitWarningThreshold`/`requestDelay` to their normal values and monitor the histograms for at least one hour.
## 7. Alert integration quick reference
- Prometheus: `ghsa_ratelimit_remaining_bucket` (from histogram) use `histogram_quantile(0.99, ...)` to trend capacity.
- VictoriaMetrics: `LAST_over_time(ghsa_ratelimit_remaining_sum[5m])` for simple last-value graphs.
- Grafana: stack remaining + used to visualise total limit per resource.
## 8. Canonical metric fallback analytics
When GitHub omits CVSS vectors/scores, the connector now assigns a deterministic canonical metric id in the form `ghsa:severity/<level>` and publishes it to Merge so severity precedence still resolves against GHSA even without CVSS data.
- Metric: `ghsa.map.canonical_metric_fallbacks` (counter) with tags `severity`, `canonical_metric_id`, `reason=no_cvss`.
- Monitor the counter alongside Merge parity checks; a sudden spike suggests GitHub is shipping advisories without vectors and warrants cross-checking downstream exporters.
- Because the canonical id feeds Merge, parity dashboards should overlay this metric to confirm fallback advisories continue to merge ahead of downstream sources when GHSA supplies more recent data.

View File

@@ -0,0 +1,122 @@
# Concelier CISA ICS Connector Operations
This runbook documents how to provision, rotate, and validate credentials for the CISA Industrial Control Systems (ICS) connector (`source:ics-cisa:*`). Follow it before enabling the connector in staging or offline installations.
## 1. Credential Provisioning
1. **Create a service mailbox** reachable by the Ops crew (shared mailbox recommended).
2. Browse to `https://public.govdelivery.com/accounts/USDHSCISA/subscriber/new` and subscribe the mailbox to the following GovDelivery topics:
- `USDHSCISA_16` — ICS-CERT advisories (legacy numbering: `ICSA-YY-###`).
- `USDHSCISA_19` — ICS medical advisories (`ICSMA-YY-###`).
- `USDHSCISA_17` — ICS alerts (`IR-ALERT-YY-###`) for completeness.
3. Complete the verification email. After confirmation, note the **personalised subscription code** included in the “Manage Preferences” link. It has the shape `code=AB12CD34EF`.
4. Store the code in the shared secret vault (or Offline Kit secrets bundle) as `concelier/sources/icscisa/govdelivery/code`.
> GovDelivery does not expose a one-time API key; the personalised code is what authenticates the RSS pull. Never commit it to git.
## 2. Feed Validation
Use the following command to confirm the feed is reachable before wiring it into Concelier (substitute `<CODE>` with the personalised value):
```bash
curl -H "User-Agent: StellaOpsConcelier/ics-cisa" \
"https://content.govdelivery.com/accounts/USDHSCISA/topics/ICS-CERT/feed.rss?format=xml&code=<CODE>"
```
If the endpoint returns HTTP 200 and an RSS payload, record the sample response under `docs/artifacts/icscisa/` (see Task `FEEDCONN-ICSCISA-02-007`). HTTP 403 or 406 usually means the subscription was not confirmed or the code was mistyped.
## 3. Configuration Snippet
Add the connector configuration to `concelier.yaml` (or equivalent environment variables):
```yaml
concelier:
sources:
icscisa:
govDelivery:
code: "${CONCELIER_ICS_CISA_GOVDELIVERY_CODE}"
topics:
- "USDHSCISA_16"
- "USDHSCISA_19"
- "USDHSCISA_17"
rssBaseUri: "https://content.govdelivery.com/accounts/USDHSCISA"
requestDelay: "00:00:01"
failureBackoff: "00:05:00"
```
Environment variable example:
```bash
export CONCELIER_SOURCES_ICSCISA_GOVDELIVERY_CODE="AB12CD34EF"
```
Concelier automatically register the host with the Source.Common HTTP allow-list when the connector assembly is loaded.
Optional tuning keys (set only when needed):
- `proxyUri` — HTTP/HTTPS proxy URL used when Akamai blocks direct pulls.
- `requestVersion` / `requestVersionPolicy` — override HTTP negotiation when the proxy requires HTTP/1.1.
- `enableDetailScrape` — toggle HTML detail fallback (defaults to true).
- `captureAttachments` — collect PDF attachments from detail pages (defaults to true).
- `detailBaseUri` — alternate host for detail enrichment if CISA changes their layout.
## 4. Seeding Without GovDelivery
If credentials are still pending, populate the connector with the community CSV dataset before enabling the live fetch:
1. Run `./scripts/fetch-ics-cisa-seed.sh` (or `.ps1`) to download the latest `CISA_ICS_ADV_*.csv` files into `seed-data/ics-cisa/`.
2. Copy the CSVs (and the generated `.sha256` files) into your Offline Kit staging area so they ship alongside the other feeds.
3. Import the kit as usual. The connector can parse the seed data for historical context, but **live GovDelivery credentials are still required** for fresh advisories.
4. Once credentials arrive, update `concelier:sources:icscisa:govDelivery:code` and re-trigger `source:ics-cisa:fetch` so the connector switches to the authorised feed.
> The CSVs are licensed under ODbL1.0 by the ICS Advisory Project. Preserve the attribution when redistributing them.
## 4. Integration Validation
1. Ensure secrets are in place and restart the Concelier workers.
2. Run a dry-run fetch/parse/map chain against an Akamai-protected topic:
```bash
CONCELIER_SOURCES_ICSCISA_GOVDELIVERY_CODE=... \
CONCELIER_SOURCES_ICSCISA_ENABLEDETAILSCRAPE=1 \
stella db jobs run source:ics-cisa:fetch --and-then source:ics-cisa:parse --and-then source:ics-cisa:map
```
3. Confirm logs contain `ics-cisa detail fetch` entries and that new documents/DTOs include attachments (see `docs/artifacts/icscisa`). Canonical advisories should expose PDF links as `references.kind == "attachment"` and affected packages should surface `primitives.semVer.exactValue` for single-version hits.
4. If Akamai blocks direct fetches, set `concelier:sources:icscisa:proxyUri` to your allow-listed egress proxy and rerun the dry-run.
## 4. Rotation & Incident Response
- Review GovDelivery access quarterly. Rotate the personalised code whenever Ops changes the service mailbox password or membership.
- Revoking the subscription in GovDelivery invalidates the code immediately; update the vault and configuration in the same change.
- If the code leaks, remove the subscription (`https://public.govdelivery.com/accounts/USDHSCISA/subscriber/manage_preferences?code=<CODE>`), resubscribe, and distribute the new value via the vault.
## 5. Offline Kit Handling
Include the personalised code in `offline-kit/secrets/concelier/icscisa.env`:
```
CONCELIER_SOURCES_ICSCISA_GOVDELIVERY_CODE=AB12CD34EF
```
The Offline Kit deployment script copies this file into the container secret directory mounted at `/run/secrets/concelier`. Ensure permissions are `600` and ownership matches the Concelier runtime user.
## 6. Telemetry & Monitoring
The connector emits metrics under the meter `StellaOps.Concelier.Connector.Ics.Cisa`. They allow operators to track Akamai fallbacks, detail enrichment health, and advisory fan-out.
- `icscisa.fetch.*` counters for `attempts`, `success`, `failures`, `not_modified`, and `fallbacks`, plus histogram `icscisa.fetch.documents` showing documents added per topic pull (tags: `concelier.source`, `icscisa.topic`).
- `icscisa.parse.*` counters for `success`/`failures` and histograms `icscisa.parse.advisories`, `icscisa.parse.attachments`, `icscisa.parse.detail_fetches` to monitor enrichment workload per feed document.
- `icscisa.detail.*` counters `success` / `failures` per advisory (tagged with `icscisa.advisory`) to alert when Akamai blocks detail pages.
- `icscisa.map.*` counters for `success`/`failures` and histograms `icscisa.map.references`, `icscisa.map.packages`, `icscisa.map.aliases` capturing canonical fan-out.
Suggested alerts:
- `increase(icscisa.fetch.failures_total[15m]) > 0` or `increase(icscisa.fetch.fallbacks_total[15m]) > 5` — sustained Akamai or proxy issues.
- `increase(icscisa.detail.failures_total[30m]) > 0` — detail enrichment breaking (potential HTML layout change).
- `histogram_quantile(0.95, rate(icscisa.map.references_bucket[1h]))` trending sharply higher — sudden advisory reference explosion worth investigating.
- Keep an eye on shared HTTP metrics (`concelier.source.http.*{concelier.source="ics-cisa"}`) for request latency and retry patterns.
## 6. Related Tasks
- `FEEDCONN-ICSCISA-02-009` (GovDelivery credential onboarding) — completed once this runbook is followed and secrets are placed in the vault.
- `FEEDCONN-ICSCISA-02-007` (document inventory) — archive the first successful RSS response and any attachment URL schema under `docs/artifacts/icscisa/`.

View File

@@ -0,0 +1,74 @@
# Concelier KISA Connector Operations
Operational guidance for the Korea Internet & Security Agency (KISA / KNVD) connector (`source:kisa:*`). Pair this with the engineering brief in `docs/dev/kisa_connector_notes.md`.
## 1. Prerequisites
- Outbound HTTPS (or mirrored cache) for `https://knvd.krcert.or.kr/`.
- Connector options defined under `concelier:sources:kisa`:
```yaml
concelier:
sources:
kisa:
feedUri: "https://knvd.krcert.or.kr/rss/securityInfo.do"
detailApiUri: "https://knvd.krcert.or.kr/rssDetailData.do"
detailPageUri: "https://knvd.krcert.or.kr/detailDos.do"
maxAdvisoriesPerFetch: 10
requestDelay: "00:00:01"
failureBackoff: "00:05:00"
```
> Ensure the URIs stay absolute—Concelier adds the `feedUri`/`detailApiUri` hosts to the HttpClient allow-list automatically.
## 2. Staging Smoke Test
1. Restart the Concelier workers so the KISA options bind.
2. Run a full connector cycle:
- CLI: `stella db jobs run source:kisa:fetch --and-then source:kisa:parse --and-then source:kisa:map`
- REST: `POST /jobs/run { "kind": "source:kisa:fetch", "chain": ["source:kisa:parse", "source:kisa:map"] }`
3. Confirm telemetry (Meter `StellaOps.Concelier.Connector.Kisa`):
- `kisa.feed.success`, `kisa.feed.items`
- `kisa.detail.success` / `.failures`
- `kisa.parse.success` / `.failures`
- `kisa.map.success` / `.failures`
- `kisa.cursor.updates`
4. Inspect logs for structured entries:
- `KISA feed returned {ItemCount}`
- `KISA fetched detail for {Idx} … category={Category}`
- `KISA mapped advisory {AdvisoryId} (severity={Severity})`
- Absence of warnings such as `document missing GridFS payload`.
5. Validate MongoDB state:
- `raw_documents.metadata` has `kisa.idx`, `kisa.category`, `kisa.title`.
- DTO store contains `schemaVersion="kisa.detail.v1"`.
- Advisories include aliases (`IDX`, CVE) and `language="ko"`.
- `source_states` entry for `kisa` shows recent `cursor.lastFetchAt`.
## 3. Production Monitoring
- **Dashboards** Add the following Prometheus/OTEL expressions:
- `rate(kisa_feed_items_total[15m])` versus `rate(concelier_source_http_requests_total{concelier_source="kisa"}[15m])`
- `increase(kisa_detail_failures_total{reason!="empty-document"}[1h])` alert at `>0`
- `increase(kisa_parse_failures_total[1h])` for storage/JSON issues
- `increase(kisa_map_failures_total[1h])` to flag schema drift
- `increase(kisa_cursor_updates_total[6h]) == 0` during active windows → warn
- **Alerts** Page when `rate(kisa_feed_success_total[2h]) == 0` while other connectors are active; back off for maintenance windows announced on `https://knvd.krcert.or.kr/`.
- **Logs** Watch for repeated warnings (`document missing`, `DTO missing`) or errors with reason tags `HttpRequestException`, `download`, `parse`, `map`.
## 4. Localisation Handling
- Hangul categories (for example `취약점정보`) flow into telemetry tags (`category=…`) and logs. Dashboards must render UTF8 and avoid transliteration.
- HTML content is sanitised before storage; translation teams can consume the `ContentHtml` field safely.
- Advisory severity remains as provided by KISA (`High`, `Medium`, etc.). Map-level failures include the severity tag for filtering.
## 5. Fixture & Regression Maintenance
- Regression fixtures: `src/StellaOps.Concelier.Connector.Kisa.Tests/Fixtures/kisa-feed.xml` and `kisa-detail.json`.
- Refresh via `UPDATE_KISA_FIXTURES=1 dotnet test src/StellaOps.Concelier.Connector.Kisa.Tests/StellaOps.Concelier.Connector.Kisa.Tests.csproj`.
- The telemetry regression (`KisaConnectorTests.Telemetry_RecordsMetrics`) will fail if counters/log wiring drifts—treat failures as gating.
## 6. Known Issues
- RSS feeds only expose the latest 10 advisories; long outages require replay via archived feeds or manual IDX seeds.
- Detail endpoint occasionally throttles; the connector honours `requestDelay` and reports failures with reason `HttpRequestException`. Consider increasing delay for weekend backfills.
- If `kisa.category` tags suddenly appear as `unknown`, verify KISA has not renamed RSS elements; update the parser fixtures before production rollout.

View File

@@ -0,0 +1,86 @@
# Concelier MSRC Connector Azure AD Onboarding Brief
_Drafted: 2025-10-15_
## 1. App registration requirements
- **Tenant**: shared StellaOps production Azure AD.
- **Application type**: confidential client (web/API) issuing client credentials.
- **API permissions**: `api://api.msrc.microsoft.com/.default` (Application). Admin consent required once.
- **Token audience**: `https://api.msrc.microsoft.com/`.
- **Grant type**: client credentials. Concelier will request tokens via `POST https://login.microsoftonline.com/{tenantId}/oauth2/v2.0/token`.
## 2. Secret/credential policy
- Maintain two client secrets (primary + standby) rotating every 90 days.
- Store secrets in the Concelier secrets vault; Offline Kit deployments must mirror the secret payloads in their encrypted store.
- Record rotation cadence in Ops runbook and update Concelier configuration (`CONCELIER__SOURCES__VNDR__MSRC__CLIENTSECRET`) ahead of expiry.
## 3. Concelier configuration sample
```yaml
concelier:
sources:
vndr.msrc:
tenantId: "<azure-tenant-guid>"
clientId: "<app-registration-client-id>"
clientSecret: "<pull from secret store>"
apiVersion: "2024-08-01"
locale: "en-US"
requestDelay: "00:00:00.250"
failureBackoff: "00:05:00"
cursorOverlapMinutes: 10
downloadCvrf: false # set true to persist CVRF ZIP alongside JSON detail
```
## 4. CVRF artefacts
- The MSRC REST payload exposes `cvrfUrl` per advisory. Current connector persists the link as advisory metadata and reference; it does **not** download the ZIP by default.
- Ops should mirror CVRF ZIPs when preparing Offline Kits so air-gapped deployments can reconcile advisories without direct internet access.
- Once Offline Kit storage guidelines are finalised, extend the connector configuration with `downloadCvrf: true` to enable automatic attachment retrieval.
### 4.1 State seeding helper
Use `tools/SourceStateSeeder` to queue historical advisories (detail JSON + optional CVRF artefacts) for replay without manual Mongo edits. Example seed file:
```json
{
"source": "vndr.msrc",
"cursor": {
"lastModifiedCursor": "2024-01-01T00:00:00Z"
},
"documents": [
{
"uri": "https://api.msrc.microsoft.com/sug/v2.0/vulnerability/ADV2024-0001",
"contentFile": "./seeds/adv2024-0001.json",
"contentType": "application/json",
"metadata": { "msrc.vulnerabilityId": "ADV2024-0001" },
"addToPendingDocuments": true
},
{
"uri": "https://download.microsoft.com/msrc/2024/ADV2024-0001.cvrf.zip",
"contentFile": "./seeds/adv2024-0001.cvrf.zip",
"contentType": "application/zip",
"status": "mapped",
"addToPendingDocuments": false
}
]
}
```
Run the helper:
```bash
dotnet run --project tools/SourceStateSeeder -- \
--connection-string "mongodb://localhost:27017" \
--database concelier \
--input seeds/msrc-backfill.json
```
Any documents marked `addToPendingDocuments` will appear in the connector cursor; `DownloadCvrf` can remain disabled if the ZIP artefact is pre-seeded.
## 5. Outstanding items
- Ops to confirm tenant/app names and provide client credentials through the secure channel.
- Connector team monitors token cache health (already implemented); validate instrumentation once Ops supplies credentials.
- Offline Kit packaging: add encrypted blob containing client credentials with rotation instructions.

View File

@@ -0,0 +1,48 @@
# NKCKI Connector Operations Guide
## Overview
The NKCKI connector ingests JSON bulletin archives from cert.gov.ru, expanding each `*.json.zip` attachment into per-vulnerability DTOs before canonical mapping. The fetch pipeline now supports cache-backed recovery, deterministic pagination, and telemetry suitable for production monitoring.
## Configuration
Key options exposed through `concelier:sources:ru-nkcki:http`:
- `maxBulletinsPerFetch` limits new bulletin downloads in a single run (default `5`).
- `maxListingPagesPerFetch` maximum listing pages visited during pagination (default `3`).
- `listingCacheDuration` minimum interval between listing fetches before falling back to cached artefacts (default `00:10:00`).
- `cacheDirectory` optional path for persisted bulletin archives used during offline or failure scenarios.
- `requestDelay` delay inserted between bulletin downloads to respect upstream politeness.
When operating in offline-first mode, set `cacheDirectory` to a writable path (e.g. `/var/lib/concelier/cache/ru-nkcki`) and pre-populate bulletin archives via the offline kit.
## Telemetry
`RuNkckiDiagnostics` emits the following metrics under meter `StellaOps.Concelier.Connector.Ru.Nkcki`:
- `nkcki.listing.fetch.attempts` / `nkcki.listing.fetch.success` / `nkcki.listing.fetch.failures`
- `nkcki.listing.pages.visited` (histogram, `pages`)
- `nkcki.listing.attachments.discovered` / `nkcki.listing.attachments.new`
- `nkcki.bulletin.fetch.success` / `nkcki.bulletin.fetch.cached` / `nkcki.bulletin.fetch.failures`
- `nkcki.entries.processed` (histogram, `entries`)
Integrate these counters into standard Concelier observability dashboards to track crawl coverage and cache hit rates.
## Archive Backfill Strategy
Bitrix pagination surfaces archives via `?PAGEN_1=n`. The connector now walks up to `maxListingPagesPerFetch` pages, deduplicating bulletin IDs and maintaining a rolling `knownBulletins` window. Backfill strategy:
1. Enumerate pages from newest to oldest, respecting `maxListingPagesPerFetch` and `listingCacheDuration` to avoid refetch storms.
2. Persist every `*.json.zip` attachment to the configured cache directory. This enables replay when listing access is temporarily blocked.
3. During archive replay, `ProcessCachedBulletinsAsync` enqueues missing documents while respecting `maxVulnerabilitiesPerFetch`.
4. For historical HTML-only advisories, collect page URLs and metadata while offline (future work: HTML and PDF extraction pipeline documented in `docs/concelier-connector-research-20251011.md`).
For large migrations, seed caches with archived zip bundles, then run fetch/parse/map cycles in chronological order to maintain deterministic outputs.
## Failure Handling
- Listing failures mark the source state with exponential backoff while attempting cache replay.
- Bulletin fetches fall back to cached copies before surfacing an error.
- Mongo integration tests rely on bundled OpenSSL 1.1 libraries (`tools/openssl/linux-x64`) to keep `Mongo2Go` operational on modern distros.
Refer to `ru-nkcki` entries in `src/StellaOps.Concelier.Connector.Ru.Nkcki/TASKS.md` for outstanding items.

View File

@@ -0,0 +1,24 @@
# Concelier OSV Connector Operations Notes
_Last updated: 2025-10-16_
The OSV connector ingests advisories from OSV.dev across OSS ecosystems. This note highlights the additional merge/export expectations introduced with the canonical metric fallback work in Sprint 4.
## 1. Canonical metric fallbacks
- When OSV omits CVSS vectors (common for CVSS v4-only payloads) the mapper now emits a deterministic canonical metric id in the form `osv:severity/<level>` and normalises the advisory severity to the same `<level>`.
- Metric: `osv.map.canonical_metric_fallbacks` (counter) with tags `severity`, `canonical_metric_id`, `ecosystem`, `reason=no_cvss`. Watch this alongside merge parity dashboards to catch spikes where OSV publishes severity-only advisories.
- Merge precedence still prefers GHSA over OSV; the shared severity-based canonical id keeps Merge/export parity deterministic even when only OSV supplies severity data.
## 2. CWE provenance
- `database_specific.cwe_ids` now populates provenance decision reasons for every mapped weakness. Expect `decisionReason="database_specific.cwe_ids"` on OSV weakness provenance and confirm exporters preserve the value.
- If OSV ever attaches `database_specific.cwe_notes`, the connector will surface the joined note string in `decisionReason` instead of the default marker.
## 3. Dashboards & alerts
- Extend existing merge dashboards with the new counter:
- Overlay `sum(osv.map.canonical_metric_fallbacks{ecosystem=~".+"})` with Merge severity overrides to confirm fallback advisories are reconciling cleanly.
- Alert when the 1-hour sum exceeds 50 for any ecosystem; baseline volume is currently <5 per day (mostly GHSA mirrors emitting CVSS v4 only).
- Exporters already surface `canonicalMetricId`; no schema change is required, but ORAS/Trivy bundles should be spot-checked after deploying the connector update.
## 4. Runbook updates
- Fixture parity suites (`osv-ghsa.*`) now assert the fallback id and provenance notes. Regenerate via `dotnet test src/StellaOps.Concelier.Connector.Osv.Tests/StellaOps.Concelier.Connector.Osv.Tests.csproj`.
- When investigating merge severity conflicts, include the fallback counter and confirm OSV advisories carry the expected `osv:severity/<level>` id before raising connector bugs.

View File

@@ -1,152 +0,0 @@
# Feedser Conflict Resolution Runbook (Sprint 3)
This runbook equips Feedser operators to detect, triage, and resolve advisory conflicts now that the Sprint 3 merge engine landed (`AdvisoryPrecedenceMerger`, merge-event hashing, and telemetry counters). It builds on the canonical rules defined in `src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md` and the metrics/logging instrumentation delivered this sprint.
---
## 1. Precedence Model (recap)
- **Default ranking:** `GHSA -> NVD -> OSV`, with distro/vendor PSIRTs outranking ecosystem feeds (`AdvisoryPrecedenceDefaults`). Use `feedser:merge:precedence:ranks` to override per source when incident response requires it.
- **Freshness override:** if a lower-ranked source is >= 48 hours newer for a freshness-sensitive field (title, summary, affected ranges, references, credits), it wins. Every override stamps `provenance[].decisionReason = freshness`.
- **Tie-breakers:** when precedence and freshness tie, the engine falls back to (1) primary source order, (2) shortest normalized text, (3) lowest stable hash. Merge-generated provenance records set `decisionReason = tie-breaker`.
- **Audit trail:** each merged advisory receives a `merge` provenance entry listing the participating sources plus a `merge_event` record with canonical before/after SHA-256 hashes.
---
## 2. Telemetry Shipped This Sprint
| Instrument | Type | Key Tags | Purpose |
|------------|------|----------|---------|
| `feedser.merge.operations` | Counter | `inputs` | Total precedence merges executed. |
| `feedser.merge.overrides` | Counter | `primary_source`, `suppressed_source`, `primary_rank`, `suppressed_rank` | Field-level overrides chosen by precedence. |
| `feedser.merge.range_overrides` | Counter | `advisory_key`, `package_type`, `primary_source`, `suppressed_source`, `primary_range_count`, `suppressed_range_count` | Package range overrides emitted by `AffectedPackagePrecedenceResolver`. |
| `feedser.merge.conflicts` | Counter | `type` (`severity`, `precedence_tie`), `reason` (`mismatch`, `primary_missing`, `equal_rank`) | Conflicts requiring operator review. |
| `feedser.merge.identity_conflicts` | Counter | `scheme`, `alias_value`, `advisory_count` | Alias collisions surfaced by the identity graph. |
### Structured logs
- `AdvisoryOverride` (EventId 1000) - logs merge suppressions with alias/provenance counts.
- `PackageRangeOverride` (EventId 1001) - logs package-level precedence decisions.
- `PrecedenceConflict` (EventId 1002) - logs mismatched severity or equal-rank scenarios.
- `Alias collision ...` (no EventId) - emitted when `feedser.merge.identity_conflicts` increments.
Expect all logs at `Information`. Ensure OTEL exporters include the scope `StellaOps.Feedser.Merge`.
---
## 3. Detection & Alerting
1. **Dashboard panels**
- `feedser.merge.conflicts` - table grouped by `type/reason`. Alert when > 0 in a 15 minute window.
- `feedser.merge.range_overrides` - stacked bar by `package_type`. Spikes highlight vendor PSIRT overrides over registry data.
- `feedser.merge.overrides` with `primary_source|suppressed_source` - catches unexpected precedence flips (e.g., OSV overtaking GHSA).
- `feedser.merge.identity_conflicts` - single-stat; alert when alias collisions occur more than once per day.
2. **Log based alerts**
- `eventId=1002` with `reason="equal_rank"` - indicates precedence table gaps; page merge owners.
- `eventId=1002` with `reason="mismatch"` - severity disagreement; open connector bug if sustained.
3. **Job health**
- `stellaops-cli db merge` exit code `1` signifies unresolved conflicts. Pipe to automation that captures logs and notifies #feedser-ops.
### Threshold updates (2025-10-12)
- `feedser.merge.conflicts` Page only when ≥ 2 events fire within 30 minutes; the synthetic conflict fixture run produces 0 conflicts, so the first event now routes to Slack for manual review instead of paging.
- `feedser.merge.overrides` Raise a warning when the 30-minute sum exceeds 10 (canonical triple yields exactly 1 summary override with `primary_source=osv`, `suppressed_source=ghsa`).
- `feedser.merge.range_overrides` Maintain the 15-minute alert at ≥ 3 but annotate dashboards that the regression triple emits a single `package_type=semver` override so ops can spot unexpected spikes.
---
## 4. Triage Workflow
1. **Confirm job context**
- `stellaops-cli db merge` (CLI) or `POST /jobs/merge:reconcile` (API) to rehydrate the merge job. Use `--verbose` to stream structured logs during triage.
2. **Inspect metrics**
- Correlate spikes in `feedser.merge.conflicts` with `primary_source`/`suppressed_source` tags from `feedser.merge.overrides`.
3. **Pull structured logs**
- Example (vector output):
```
jq 'select(.EventId.Name=="PrecedenceConflict") | {advisory: .State[0].Value, type: .ConflictType, reason: .Reason, primary: .PrimarySources, suppressed: .SuppressedSources}' stellaops-feedser.log
```
4. **Review merge events**
- `mongosh`:
```javascript
use feedser;
db.merge_event.find({ advisoryKey: "CVE-2025-1234" }).sort({ mergedAt: -1 }).limit(5);
```
- Compare `beforeHash` vs `afterHash` to confirm the merge actually changed canonical output.
5. **Interrogate provenance**
- `db.advisories.findOne({ advisoryKey: "CVE-2025-1234" }, { title: 1, severity: 1, provenance: 1, "affectedPackages.provenance": 1 })`
- Check `provenance[].decisionReason` values (`precedence`, `freshness`, `tie-breaker`) to understand why the winning field was chosen.
---
## 5. Conflict Classification Matrix
| Signal | Likely Cause | Immediate Action |
|--------|--------------|------------------|
| `reason="mismatch"` with `type="severity"` | Upstream feeds disagree on CVSS vector/severity. | Verify which feed is freshest; if correctness is known, adjust connector mapping or precedence override. |
| `reason="primary_missing"` | Higher-ranked source lacks the field entirely. | Backfill connector data or temporarily allow lower-ranked source via precedence override. |
| `reason="equal_rank"` | Two feeds share the same precedence rank (custom config or missing entry). | Update `feedser:merge:precedence:ranks` to break the tie; restart merge job. |
| Rising `feedser.merge.range_overrides` for a package type | Vendor PSIRT now supplies richer ranges. | Validate connectors emit `decisionReason="precedence"` and update dashboards to treat registry ranges as fallback. |
| `feedser.merge.identity_conflicts` > 0 | Alias scheme mapping produced collisions (duplicate CVE <-> advisory pairs). | Inspect `Alias collision` log payload; reconcile the alias graph by adjusting connector alias output. |
---
## 6. Resolution Playbook
1. **Connector data fix**
- Re-run the offending connector stages (`stellaops-cli db fetch --source ghsa --stage map` etc.).
- Once fixed, rerun merge and verify `decisionReason` reflects `freshness` or `precedence` as expected.
2. **Temporary precedence override**
- Edit `etc/feedser.yaml`:
```yaml
feedser:
merge:
precedence:
ranks:
osv: 1
ghsa: 0
```
- Restart Feedser workers; confirm tags in `feedser.merge.overrides` show the new ranks.
- Document the override with expiry in the change log.
3. **Alias remediation**
- Update connector mapping rules to weed out duplicate aliases (e.g., skip GHSA aliases that mirror CVE IDs).
- Flush cached alias graphs if necessary (`db.alias_graph.drop()` is destructive-coordinate with Storage before issuing).
4. **Escalation**
- If override metrics spike due to upstream regression, open an incident with Security Guild, referencing merge logs and `merge_event` IDs.
---
## 7. Validation Checklist
- [ ] Merge job rerun returns exit code `0`.
- [ ] `feedser.merge.conflicts` baseline returns to zero after corrective action.
- [ ] Latest `merge_event` entry shows expected hash delta.
- [ ] Affected advisory document shows updated `provenance[].decisionReason`.
- [ ] Ops change log updated with incident summary, config overrides, and rollback plan.
---
## 8. Reference Material
- Canonical conflict rules: `src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md`.
- Merge engine internals: `src/StellaOps.Feedser.Merge/Services/AdvisoryPrecedenceMerger.cs`.
- Metrics definitions: `src/StellaOps.Feedser.Merge/Services/AdvisoryMergeService.cs` (identity conflicts) and `AdvisoryPrecedenceMerger`.
- Storage audit trail: `src/StellaOps.Feedser.Merge/Services/MergeEventWriter.cs`, `src/StellaOps.Feedser.Storage.Mongo/MergeEvents`.
Keep this runbook synchronized with future sprint notes and update alert thresholds as baseline volumes change.
---
## 9. Synthetic Regression Fixtures
- **Locations** Canonical conflict snapshots now live at `src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/conflict-ghsa.canonical.json`, `src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/conflict-nvd.canonical.json`, and `src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/conflict-osv.canonical.json`.
- **Validation commands** To regenerate and verify the fixtures offline, run:
```bash
dotnet test src/StellaOps.Feedser.Source.Ghsa.Tests/StellaOps.Feedser.Source.Ghsa.Tests.csproj --filter GhsaConflictFixtureTests
dotnet test src/StellaOps.Feedser.Source.Nvd.Tests/StellaOps.Feedser.Source.Nvd.Tests.csproj --filter NvdConflictFixtureTests
dotnet test src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj --filter OsvConflictFixtureTests
dotnet test src/StellaOps.Feedser.Merge.Tests/StellaOps.Feedser.Merge.Tests.csproj --filter MergeAsync_AppliesCanonicalRulesAndPersistsDecisions
```
- **Expected signals** The triple produces one freshness-driven summary override (`primary_source=osv`, `suppressed_source=ghsa`) and one range override for the npm SemVer package while leaving `feedser.merge.conflicts` at zero. Use these values as the baseline when tuning dashboards or load-testing alert pipelines.

View File

@@ -6,12 +6,12 @@ _Last updated: 2025-10-11_
The SemVer style migration populates the new `normalizedVersions` field on advisory documents and ensures
provenance `decisionReason` values are preserved during future reads. The migration is idempotent and only
runs when the feature flag `feedser:storage:enableSemVerStyle` is enabled.
runs when the feature flag `concelier:storage:enableSemVerStyle` is enabled.
## Preconditions
1. **Review configuration** set `feedser.storage.enableSemVerStyle` to `true` on all Feedser services.
2. **Confirm batch size** adjust `feedser.storage.backfillBatchSize` if you need smaller batches for older
1. **Review configuration** set `concelier.storage.enableSemVerStyle` to `true` on all Concelier services.
2. **Confirm batch size** adjust `concelier.storage.backfillBatchSize` if you need smaller batches for older
deployments (default: `250`).
3. **Back up** capture a fresh snapshot of the `advisory` collection or a full MongoDB backup.
4. **Staging dry-run** enable the flag in a staging environment and observe the migration output before
@@ -19,7 +19,7 @@ runs when the feature flag `feedser:storage:enableSemVerStyle` is enabled.
## Execution
No manual command is required. After deploying the configuration change, restart the Feedser WebService or
No manual command is required. After deploying the configuration change, restart the Concelier WebService or
any component that hosts the Mongo migration runner. During startup you will see log entries similar to:
```
@@ -27,7 +27,7 @@ Applying Mongo migration 20251011-semver-style-backfill: Populate advisory.norma
Mongo migration 20251011-semver-style-backfill applied
```
The migration reads advisories in batches (`feedser.storage.backfillBatchSize`) and writes flattened
The migration reads advisories in batches (`concelier.storage.backfillBatchSize`) and writes flattened
`normalizedVersions` arrays. Existing documents without SemVer ranges remain untouched.
## Post-checks
@@ -39,12 +39,12 @@ The migration reads advisories in batches (`feedser.storage.backfillBatchSize`)
You should see `advisory_normalizedVersions_pkg_scheme_type` and `advisory_normalizedVersions_value`.
2. Spot check a few advisories to confirm the top-level `normalizedVersions` array exists and matches
the embedded package data.
3. Run `dotnet test` for `StellaOps.Feedser.Storage.Mongo.Tests` (optional but recommended) in CI to confirm
3. Run `dotnet test` for `StellaOps.Concelier.Storage.Mongo.Tests` (optional but recommended) in CI to confirm
the storage suite passes with the feature flag enabled.
## Rollback
Set `feedser.storage.enableSemVerStyle` back to `false` and redeploy. The migration will be skipped on
Set `concelier.storage.enableSemVerStyle` back to `false` and redeploy. The migration will be skipped on
subsequent startups. You can leave the populated `normalizedVersions` arrays in place; they are ignored when
the feature flag is off. If you must remove them entirely, restore from the backup captured during
preparation.

View File

@@ -15,7 +15,7 @@ Audit events share the `StellaOps.Cryptography.Audit.AuthEventRecord` contract.
- `Client``AuthEventClient` with client identifier, display name, and originating provider/plugin.
- `Scopes` — granted or requested OAuth scopes (sorted before emission).
- `Network``AuthEventNetwork` with remote address, forwarded headers, and user agent string (all treated as PII).
- `Properties` — additional `AuthEventProperty` entries for context-specific details (lockout durations, policy decisions, retries, etc.).
- `Properties` — additional `AuthEventProperty` entries for context-specific details (lockout durations, policy decisions, retries, `request.tampered`/`request.unexpected_parameter`, `bootstrap.invite_token`, etc.).
## Data Classifications
@@ -33,7 +33,13 @@ Event names follow dotted notation:
- `authority.password.grant` — password grant handled by OpenIddict.
- `authority.client_credentials.grant` — client credential grant handling.
- `authority.token.tamper` — suspicious `/token` request detected (unexpected parameters or manipulated payload).
- `authority.bootstrap.user` and `authority.bootstrap.client` — bootstrap API operations.
- `authority.bootstrap.invite.created` — operator created a bootstrap invite.
- `authority.bootstrap.invite.consumed` — invite consumed during user/client provisioning.
- `authority.bootstrap.invite.expired` — invite expired without being used.
- `authority.bootstrap.invite.rejected` — invite was rejected (invalid, mismatched provider/target, or already consumed).
- `authority.token.replay.suspected` — replay heuristics detected a token being used from a new device fingerprint.
- Future additions should preserve the `authority.<surface>.<action>` pattern to keep filtering deterministic.
## Persistence

View File

@@ -14,7 +14,7 @@
|-----------------|-------------|----------------|
| Token issuance APIs (`/token`, `/authorize`) | OAuth/OIDC endpoints mediated by OpenIddict | CLI, UI, automation agents |
| Bootstrap channel | Initial admin invite + bootstrap CLI workflow | Platform operators |
| Revocation bundle | Offline JSON + detached JWS consumed by agents | Feedser, Agents, Zastava |
| Revocation bundle | Offline JSON + detached JWS consumed by agents | Concelier, Agents, Zastava |
| Plug-in manifests | Standard plug-in configuration and password policy overrides | Operators, DevOps |
| Signing keys | ES256 signing keys backing tokens and revocation manifests | Security Guild, HSM/KeyOps |
| Audit telemetry | Structured login/audit stream persisted to Mongo/observability stack | SOC, SecOps |
@@ -68,7 +68,7 @@ flowchart LR
end
subgraph Distribution
OFFKIT[Offline Kit Bundle]
AGENT[Authorized Agent / Feedser]
AGENT[Authorized Agent / Concelier]
end
OPS -->|Bootstrap CLI (`stellaops auth bootstrap`)| AUTH
AUTH -->|One-time invite + Argon2 hash| STORE
@@ -82,9 +82,9 @@ flowchart LR
| Threat | STRIDE Vector | Surface | Risk (L×I) | Existing Controls | Gaps / Actions | Owner |
|--------|---------------|---------|------------|-------------------|----------------|-------|
| Spoofed revocation bundle | Spoofing | TB5 — Authority ↔️ Agents | Med×High | Detached JWS signature (planned), offline kit checksums | Finalise signing key registry & verification script (SEC4.B/SEC4.HOST); add bundle freshness requirement | Security Guild (follow-up: **SEC5.B**) |
| Parameter tampering on `/token` | Tampering | TB1 — Public ingress | Med×High | ASP.NET model validation, OpenIddict, rate limiter (CORE8.RL) | Add audit coverage for tampered inputs, align correlation IDs with SOC (SEC2.A/SEC2.B) | Security Guild + Authority Core (follow-up: **SEC5.C**) |
| Bootstrap invite replay | Repudiation | TB4 — Operator CLI ↔️ Authority | Low×High | One-time bootstrap tokens, Argon2id hashing on creation | Enforce invite expiration + audit trail for unused invites | Security Guild (follow-up: **SEC5.D**) |
| Token replay by stolen agent | Information Disclosure | TB5 | Med×High | Planned revocation bundles, optional mTLS | Require agent binding (device fingerprint) and enforce revocation grace window alerts | Security Guild + Zastava (follow-up: **SEC5.E**) |
| Parameter tampering on `/token` | Tampering | TB1 — Public ingress | Med×High | ASP.NET model validation, OpenIddict, rate limiter (CORE8.RL) | Tampered requests emit `authority.token.tamper` audit events (`request.tampered`, unexpected parameter names) correlating with `/token` outcomes (SEC5.C) | Security Guild + Authority Core (follow-up: **SEC5.C**) |
| Bootstrap invite replay | Repudiation | TB4 — Operator CLI ↔️ Authority | Low×High | One-time bootstrap tokens, Argon2id hashing on creation | Invites expire automatically and emit audit events on consumption/expiration (SEC5.D) | Security Guild |
| Token replay by stolen agent | Information Disclosure | TB5 | Med×High | Signed revocation bundles, device fingerprint heuristics, optional mTLS | Monitor revocation acknowledgement latency via Zastava and tune replay alerting thresholds | Security Guild + Zastava (follow-up: **SEC5.E**) |
| Privilege escalation via plug-in override | Elevation of Privilege | TB3 — Plug-in sandbox | Med×High | Signed plug-ins, restart-only loading, configuration validation | Add static analysis on manifest overrides + runtime warning when policy weaker than host | Security Guild + DevOps (follow-up: **SEC5.F**) |
| Offline bundle tampering | Tampering | Distribution | Low×High | SHA256 manifest, signed bundles (planned) | Add supply-chain attestation for Offline Kit, publish verification CLI in docs | Security Guild + Ops (follow-up: **SEC5.G**) |
| Failure to log denied tokens | Repudiation | TB2 — Authority ↔️ Mongo | Med×Med | Serilog structured events (partial), Mongo persistence path (planned) | Finalise audit schema (SEC2.A) and ensure `/token` denies include subject/client/IP fields | Security Guild + Authority Core (follow-up: **SEC5.H**) |
@@ -98,7 +98,7 @@ Risk scoring uses qualitative scale (Low/Med/High) for likelihood × impact; mit
| SEC5.B | Spoofed revocation bundle | Complete libsodium/Core signing integration and ship revocation verification script. | Security Guild + Authority Core |
| SEC5.C | Parameter tampering on `/token` | Finalise audit contract (`SEC2.A`) and add request tamper logging. | Security Guild + Authority Core |
| SEC5.D | Bootstrap invite replay | Implement expiry enforcement + audit coverage for unused bootstrap invites. | Security Guild |
| SEC5.E | Token replay by stolen agent | Document device binding requirements and create detector for stale revocation acknowledgements. | Security Guild + Zastava |
| SEC5.E | Token replay by stolen agent | Coordinate Zastava alerting with the new device fingerprint heuristics and surface stale revocation acknowledgements. | Security Guild + Zastava |
| SEC5.F | Plug-in override escalation | Static analysis of plug-in manifests; warn on weaker password policy overrides. | Security Guild + DevOps |
| SEC5.G | Offline bundle tampering | Extend Offline Kit build to include attested manifest + verification CLI sample. | Security Guild + Ops |
| SEC5.H | Failure to log denied tokens | Ensure audit persistence for all `/token` denials with correlation IDs. | Security Guild + Authority Core |

View File

@@ -0,0 +1,76 @@
# StellaOps Authority Rate Limit Guidance
StellaOps Authority applies fixed-window rate limiting to critical endpoints so that brute-force and burst traffic are throttled before they can exhaust downstream resources. This guide complements the lockout policy documentation and captures the recommended defaults, override scenarios, and monitoring practices for `/token`, `/authorize`, and `/internal/*` routes.
## Configuration Overview
Rate limits live under `security.rateLimiting` in `authority.yaml` (and map to the same hierarchy for environment variables). Each endpoint exposes:
- `enabled` &mdash; toggles the limiter.
- `permitLimit` &mdash; maximum requests per fixed window.
- `window` &mdash; window duration expressed as an ISO-8601 timespan (e.g., `00:01:00`).
- `queueLimit` &mdash; number of requests allowed to queue when the window is exhausted.
```yaml
security:
rateLimiting:
token:
enabled: true
permitLimit: 30
window: 00:01:00
queueLimit: 0
authorize:
enabled: true
permitLimit: 60
window: 00:01:00
queueLimit: 10
internal:
enabled: false
permitLimit: 5
window: 00:01:00
queueLimit: 0
```
When limits trigger, middleware decorates responses with `Retry-After` headers and log tags (`authority.endpoint`, `authority.client_id`, `authority.remote_ip`) so operators can correlate events with clients and source IPs.
Environment overrides follow the same hierarchy. For example:
```
STELLAOPS_AUTHORITY__SECURITY__RATELIMITING__TOKEN__PERMITLIMIT=60
STELLAOPS_AUTHORITY__SECURITY__RATELIMITING__TOKEN__WINDOW=00:01:00
```
## Recommended Profiles
| Scenario | permitLimit | window | queueLimit | Notes |
|----------|-------------|--------|------------|-------|
| Default production | 30 | 60s | 0 | Balances anonymous quota (33 scans/day) with headroom for tenant bursts. |
| High-trust clustered IPs | 60 | 60s | 5 | Requires WAF allowlist + alert `aspnetcore_rate_limiting_rejections_total{limiter="authority-token"} <= 1%` sustained. |
| Air-gapped lab | 10 | 120s | 0 | Lower concurrency reduces noise when running from shared bastion hosts. |
| Incident lockdown | 5 | 300s | 0 | Pair with credential lockout limit of 3 attempts and SOC paging for each denial. |
### Lockout Interplay
- Rate limiting throttles by IP/client; lockout policies apply per subject. Keep both enabled.
- During lockdown scenarios, reduce `security.lockout.maxFailures` alongside the rate limits above so that subjects face quicker escalation.
- Map support playbooks to the observed `Retry-After` value: anything above 120 seconds should trigger manual investigation before re-enabling clients.
## Monitoring and Alerts
1. **Metrics**
- `aspnetcore_rate_limiting_rejections_total{limiter="authority-token"}` for `/token`.
- `aspnetcore_rate_limiting_rejections_total{limiter="authority-authorize"}` for `/authorize`.
- Custom counters derived from the structured log tags (`authority.remote_ip`, `authority.client_id`).
2. **Dashboards**
- Requests vs. rejections per endpoint.
- Top offending clients/IP ranges in the current window.
- Heatmap of retry-after durations to spot persistent throttling.
3. **Alerts**
- Notify SOC when 429 rates exceed 25% for five consecutive minutes on any limiter.
- Trigger client-specific alerts when a single client_id produces >100 throttle events/hour.
## Operational Checklist
- Validate updated limits in staging before production rollout; smoke-test with representative workload.
- When raising limits, confirm audit events continue to capture `authority.client_id`, `authority.remote_ip`, and correlation IDs for throttle responses.
- Document any overrides in the change log, including justification and expiry review date.

View File

@@ -12,14 +12,14 @@
"id": "7ad4f3d2c21b461d9b3420e1151be9c4",
"category": "token",
"tokenType": "access_token",
"clientId": "feedser-cli",
"clientId": "concelier-cli",
"subjectId": "user:ops-admin",
"reason": "compromised",
"reasonDescription": "Access token reported by SOC automation run R-2045.",
"revokedAt": "2025-10-12T14:32:05Z",
"scopes": [
"feedser:export",
"feedser:jobs"
"concelier:export",
"concelier:jobs"
],
"fingerprint": "AD35E719C12204D7E7C92ED3F6DEBF0A44642D41AAF94233F9A47E183F4C5F18",
"metadata": {

View File

@@ -1,6 +1,6 @@
# Authority Revocation Bundle
The Authority service exports revocation information as an offline-friendly JSON document plus a detached JWS signature. Operators can mirror the bundle alongside Feedser exports to ensure air-gapped scanners receive the latest token, subject, and client revocations.
The Authority service exports revocation information as an offline-friendly JSON document plus a detached JWS signature. Operators can mirror the bundle alongside Concelier exports to ensure air-gapped scanners receive the latest token, subject, and client revocations.
## File layout
@@ -43,6 +43,7 @@ Consumers MUST treat the combination of `schemaVersion` and `sequence` as a mono
{
"alg": "ES256",
"kid": "{signingKeyId}",
"provider": "{providerName}",
"typ": "application/vnd.stellaops.revocation-bundle+jws",
"b64": false,
"crit": ["b64"]
@@ -54,8 +55,28 @@ Verification steps:
1. Validate `revocation-bundle.json` against the schema.
2. Re-compute SHA-256 and compare with `.sha256` (if present).
3. Resolve the signing key from JWKS (`/.well-known/jwks.json`) or the offline key bundle.
4. Verify the detached JWS using the stored signing key (example tooling coming with `stella auth revoke verify`).
3. Resolve the signing key from JWKS (`/.well-known/jwks.json`) or the offline key bundle, preferring the provider declared in the JWS header (`provider` falls back to `default`).
4. Verify the detached JWS using the resolved provider. The CLI mirrors Authority resolution, so builds compiled with `StellaOpsCryptoSodium=true` automatically use the libsodium provider when advertised; otherwise verification downgrades to the managed fallback.
### CLI verification workflow
Use the bundled CLI command before distributing a bundle:
```bash
stellaops auth revoke verify \
--bundle artifacts/revocation-bundle.json \
--signature artifacts/revocation-bundle.json.jws \
--key etc/authority/signing/authority-public.pem \
--verbose
```
The verifier performs three checks:
1. Prints the computed digest in `sha256:<hex>` format. Compare it with the exported `.sha256` artefact.
2. Confirms the detached JWS header advertises `b64: false`, captures the provider hint, and that the algorithm matches the Authority configuration (ES256 unless overridden).
3. Registers the supplied PEM key with the crypto provider registry and validates the signature (falling back to the managed provider when the hinted provider is unavailable).
A zero exit code means the bundle is ready for mirroring/import. Non-zero codes signal missing arguments, malformed JWS payloads, or signature mismatches; regenerate or re-sign the bundle before distribution.
## Example
@@ -64,7 +85,7 @@ The repository contains an [example bundle](revocation-bundle-example.json) demo
## Operations Quick Reference
- `stella auth revoke export` emits a canonical JSON bundle, `.sha256` digest, and detached JWS signature in one command. Use `--output` to write into your mirror staging directory.
- `stella auth revoke verify` validates a bundle using cached JWKS or an offline PEM key and reports digest mismatches before distribution.
- `stella auth revoke verify` validates a bundle using cached JWKS or an offline PEM key, honours the `provider` metadata embedded in the signature, and reports digest mismatches before distribution.
- `POST /internal/revocations/export` provides the same payload for orchestrators that already talk to the bootstrap API.
- `POST /internal/signing/rotate` rotates JWKS material without downtime; always export a fresh bundle afterward so downstream mirrors receive signatures from the new `kid`.
- Offline Kit automation should mirror `revocation-bundle.json*` alongside Feedser exports so agents ingest revocations during the same sync pass.
- Offline Kit automation should mirror `revocation-bundle.json*` alongside Concelier exports so agents ingest revocations during the same sync pass.

View File

@@ -1,23 +1,23 @@
# Feedser configuration template for StellaOps deployments.
# Copy to ../etc/feedser.yaml (relative to the web service content root)
# Concelier configuration template for StellaOps deployments.
# Copy to ../etc/concelier.yaml (relative to the web service content root)
# and adjust the values to match your environment. Environment variables
# (prefixed with FEEDSER_) override these settings at runtime.
# (prefixed with CONCELIER_) override these settings at runtime.
storage:
driver: mongo
# Mongo connection string. Use SRV URI or standard connection string.
dsn: "mongodb://feedser:feedser@mongo:27017/feedser?authSource=admin"
# Optional database name; defaults to the name embedded in the DSN or 'feedser'.
database: "feedser"
dsn: "mongodb://concelier:concelier@mongo:27017/concelier?authSource=admin"
# Optional database name; defaults to the name embedded in the DSN or 'concelier'.
database: "concelier"
# Mongo command timeout in seconds.
commandTimeoutSeconds: 30
plugins:
# Feedser resolves plug-ins relative to the content root; override as needed.
# Concelier resolves plug-ins relative to the content root; override as needed.
baseDirectory: ".."
directory: "PluginBinaries"
searchPatterns:
- "StellaOps.Feedser.Plugin.*.dll"
- "StellaOps.Concelier.Plugin.*.dll"
telemetry:
enabled: true
@@ -25,7 +25,7 @@ telemetry:
enableMetrics: false
enableLogging: true
minimumLogLevel: "Information"
serviceName: "stellaops-feedser"
serviceName: "stellaops-concelier"
# Configure OTLP endpoint when shipping traces/metrics/logs out-of-band.
otlpEndpoint: ""
# Optional headers for OTLP exporters, for example authentication tokens.
@@ -38,7 +38,7 @@ telemetry:
authority:
enabled: false
# Temporary rollout flag. When true, Feedser logs anonymous access but does not fail requests
# Temporary rollout flag. When true, Concelier logs anonymous access but does not fail requests
# without tokens. Set to false before 2025-12-31 UTC to enforce authentication fully.
allowAnonymousFallback: true
# Issuer advertised by StellaOps Authority (e.g. https://authority.stella-ops.local).
@@ -49,16 +49,16 @@ authority:
backchannelTimeoutSeconds: 30
tokenClockSkewSeconds: 60
audiences:
- "api://feedser"
- "api://concelier"
requiredScopes:
- "feedser.jobs.trigger"
# Outbound credentials Feedser can use to call Authority (client credentials flow).
clientId: "feedser-jobs"
- "concelier.jobs.trigger"
# Outbound credentials Concelier can use to call Authority (client credentials flow).
clientId: "concelier-jobs"
# Prefer storing the secret outside of the config file. Provide either clientSecret or clientSecretFile.
clientSecret: ""
clientSecretFile: ""
clientScopes:
- "feedser.jobs.trigger"
- "concelier.jobs.trigger"
resilience:
# Enable deterministic retry/backoff when Authority is briefly unavailable.
enableRetries: true
@@ -83,3 +83,15 @@ sources:
failureBackoff: "00:05:00"
rateLimitWarningThreshold: 500
secondaryRateLimitBackoff: "00:02:00"
cve:
baseEndpoint: "https://cveawg.mitre.org/api/"
apiOrg: ""
apiUser: ""
apiKey: ""
# Optional mirror used when credentials are unavailable.
seedDirectory: "./seed-data/cve"
pageSize: 200
maxPagesPerFetch: 5
initialBackfill: "30.00:00:00"
requestDelay: "00:00:00.250"
failureBackoff: "00:10:00"

View File

@@ -0,0 +1,38 @@
param(
[string]$Destination = "$(Join-Path (Split-Path -Parent $PSCommandPath) '..' | Resolve-Path)/seed-data/ics-cisa"
)
$ErrorActionPreference = 'Stop'
New-Item -Path $Destination -ItemType Directory -Force | Out-Null
Function Write-Info($Message) { Write-Host "[ics-seed] $Message" }
Function Write-ErrorLine($Message) { Write-Host "[ics-seed][error] $Message" -ForegroundColor Red }
Function Download-File($Url, $Path) {
Write-Info "Downloading $(Split-Path $Path -Leaf)"
Invoke-WebRequest -Uri $Url -OutFile $Path -UseBasicParsing
$hash = Get-FileHash -Path $Path -Algorithm SHA256
$hash.Hash | Out-File -FilePath "$Path.sha256" -Encoding ascii
}
$base = 'https://raw.githubusercontent.com/icsadvprj/ICS-Advisory-Project/main/ICS-CERT_ADV'
$master = 'CISA_ICS_ADV_Master.csv'
$snapshot = 'CISA_ICS_ADV_2025_10_09.csv'
Write-Info 'Fetching ICS advisories seed data (ODbL v1.0)'
Download-File "$base/$master" (Join-Path $Destination $master)
Download-File "$base/$snapshot" (Join-Path $Destination $snapshot)
$medicalUrl = 'https://raw.githubusercontent.com/batarr22/ICSMA_CSV/main/ICSMA_CSV_4-20-2023.xlsx'
$medicalFile = 'ICSMA_CSV_4-20-2023.xlsx'
Write-Info 'Fetching community ICSMA snapshot'
try {
Download-File $medicalUrl (Join-Path $Destination $medicalFile)
}
catch {
Write-ErrorLine "Unable to download $medicalFile (optional): $_"
Remove-Item (Join-Path $Destination $medicalFile) -ErrorAction SilentlyContinue
}
Write-Info "Seed data ready in $Destination"
Write-Info 'Remember: data is licensed under ODbL v1.0 (see seed README).'

View File

@@ -0,0 +1,38 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
DEST_DIR="${1:-$ROOT_DIR/seed-data/ics-cisa}"
mkdir -p "$DEST_DIR"
info() { printf "[ics-seed] %s\n" "$*"; }
error() { printf "[ics-seed][error] %s\n" "$*" >&2; }
download() {
local url="$1"
local target="$2"
info "Downloading $(basename "$target")"
curl -fL "$url" -o "$target"
sha256sum "$target" > "$target.sha256"
}
BASE="https://raw.githubusercontent.com/icsadvprj/ICS-Advisory-Project/main/ICS-CERT_ADV"
MASTER_FILE="CISA_ICS_ADV_Master.csv"
SNAPSHOT_2025="CISA_ICS_ADV_2025_10_09.csv"
info "Fetching ICS advisories seed data (ODbL v1.0)"
download "$BASE/$MASTER_FILE" "$DEST_DIR/$MASTER_FILE"
download "$BASE/$SNAPSHOT_2025" "$DEST_DIR/$SNAPSHOT_2025"
MEDICAL_URL="https://raw.githubusercontent.com/batarr22/ICSMA_CSV/main/ICSMA_CSV_4-20-2023.xlsx"
MEDICAL_FILE="ICSMA_CSV_4-20-2023.xlsx"
info "Fetching community ICSMA snapshot"
if curl -fL "$MEDICAL_URL" -o "$DEST_DIR/$MEDICAL_FILE"; then
sha256sum "$DEST_DIR/$MEDICAL_FILE" > "$DEST_DIR/$MEDICAL_FILE.sha256"
else
error "Unable to download $MEDICAL_FILE (optional)."
rm -f "$DEST_DIR/$MEDICAL_FILE"
fi
info "Seed data ready in $DEST_DIR"
info "Remember: data is licensed under ODbL v1.0 (see seed README)."

View File

@@ -10,9 +10,9 @@ $env:UPDATE_APPLE_FIXTURES = "1"
Push-Location $rootDir
try {
$sentinel = Join-Path $rootDir "src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/.update-apple-fixtures"
$sentinel = Join-Path $rootDir "src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/.update-apple-fixtures"
New-Item -ItemType File -Path $sentinel -Force | Out-Null
dotnet test "src\StellaOps.Feedser.Source.Vndr.Apple.Tests\StellaOps.Feedser.Source.Vndr.Apple.Tests.csproj" @Args
dotnet test "src\StellaOps.Concelier.Connector.Vndr.Apple.Tests\StellaOps.Concelier.Connector.Vndr.Apple.Tests.csproj" @Args
}
finally {
Pop-Location

View File

@@ -10,5 +10,5 @@ else
export WSLENV="UPDATE_APPLE_FIXTURES/up"
fi
touch "$ROOT_DIR/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/.update-apple-fixtures"
( cd "$ROOT_DIR" && dotnet test "src/StellaOps.Feedser.Source.Vndr.Apple.Tests/StellaOps.Feedser.Source.Vndr.Apple.Tests.csproj" "$@" )
touch "$ROOT_DIR/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/.update-apple-fixtures"
( cd "$ROOT_DIR" && dotnet test "src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/StellaOps.Concelier.Connector.Vndr.Apple.Tests.csproj" "$@" )

View File

@@ -6,4 +6,4 @@ Param(
$Root = Split-Path -Parent $PSScriptRoot
$env:UPDATE_GOLDENS = "1"
dotnet test (Join-Path $Root "src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj") @RestArgs
dotnet test (Join-Path $Root "src/StellaOps.Concelier.Models.Tests/StellaOps.Concelier.Models.Tests.csproj") @RestArgs

View File

@@ -5,4 +5,4 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
export UPDATE_GOLDENS=1
dotnet test "$ROOT_DIR/src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj" "$@"
dotnet test "$ROOT_DIR/src/StellaOps.Concelier.Models.Tests/StellaOps.Concelier.Models.Tests.csproj" "$@"

View File

@@ -0,0 +1,52 @@
# CERT-Bund Offline Kit Seed Data
This directory stores **offline snapshots** for the CERT-Bund connector.
The artefacts mirror the public JSON search and export endpoints so
airgapped deployments can hydrate the connector without contacting the
portal.
> ⚠️ **Distribution notice** CERT-Bund advisories are published by BSI
> (Federal Office for Information Security, Germany). Review the portal
> terms of use before redistributing the snapshots. Always keep the JSON
> payloads and accompanying SHA-256 sums together.
## Recommended layout
```
seed-data/cert-bund/
├── search/ # paginated search JSON files
│   ├── certbund-search-page-00.json
│   └── …
├── export/ # yearly export JSON files
│   ├── certbund-export-2014.json
│   └── …
├── manifest/
│   └── certbund-offline-manifest.json
└── certbund-offline-manifest.sha256
```
Use `certbund-offline-manifest.json` to feed the Offline Kit build: every
entry contains `source`, `from`, `to`, `sha256`, `capturedAt`, and the
relative file path. The manifest is deterministic when regenerated with
the tooling described below.
## Tooling
Run the helper under `tools/` to capture fresh snapshots or regenerate
the manifest:
```
python tools/certbund_offline_snapshot.py --output seed-data/cert-bund
```
See the connector operations guide
(`docs/ops/concelier-certbund-operations.md`) for detailed usage,
including how to provide cookies/tokens when the portal requires manual
authentication.
## Git hygiene
- JSON payloads and checksums are **ignored by Git**. Generate them
locally when preparing an Offline Kit bundle.
- Commit documentation, scripts, and manifest templates only never the
exported advisory data itself.

View File

@@ -0,0 +1,147 @@
{
"dataType": "CVE_RECORD",
"dataVersion": "5.1",
"cveMetadata": {
"cveId": "CVE-2024-4567",
"assignerOrgId": "b15e7b5b-3da4-40ae-a43c-f7aa60e62599",
"state": "PUBLISHED",
"assignerShortName": "Wordfence",
"dateReserved": "2024-05-06T19:34:14.071Z",
"datePublished": "2024-05-09T20:03:38.213Z",
"dateUpdated": "2024-08-01T20:47:40.724Z"
},
"containers": {
"cna": {
"providerMetadata": {
"orgId": "b15e7b5b-3da4-40ae-a43c-f7aa60e62599",
"shortName": "Wordfence",
"dateUpdated": "2024-05-09T20:03:38.213Z"
},
"affected": [
{
"vendor": "themifyme",
"product": "Themify Shortcodes",
"versions": [
{
"version": "*",
"status": "affected",
"lessThanOrEqual": "2.0.9",
"versionType": "semver"
}
],
"defaultStatus": "unaffected"
}
],
"descriptions": [
{
"lang": "en",
"value": "The Themify Shortcodes plugin for WordPress is vulnerable to Stored Cross-Site Scripting via the plugin's themify_button shortcode in all versions up to, and including, 2.0.9 due to insufficient input sanitization and output escaping on user supplied attributes. This makes it possible for authenticated attackers, with contributor-level access and above, to inject arbitrary web scripts in pages that will execute whenever a user accesses an injected page."
}
],
"title": "Themify Shortcodes <= 2.0.9 - Authenticated (Contributor+) Stored Cross-Site Scripting via themify_button Shortcode",
"references": [
{
"url": "https://www.wordfence.com/threat-intel/vulnerabilities/id/c63ff9d7-6a14-4186-8550-4e5c50855e7f?source=cve"
},
{
"url": "https://plugins.trac.wordpress.org/changeset/3082885/themify-shortcodes"
}
],
"problemTypes": [
{
"descriptions": [
{
"lang": "en",
"description": "CWE-79 Improper Neutralization of Input During Web Page Generation ('Cross-site Scripting')"
}
]
}
],
"metrics": [
{
"cvssV3_1": {
"version": "3.1",
"vectorString": "CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:C/C:L/I:L/A:N",
"baseScore": 6.4,
"baseSeverity": "MEDIUM"
}
}
],
"credits": [
{
"lang": "en",
"type": "finder",
"value": "Francesco Carlucci"
}
],
"timeline": [
{
"time": "2024-05-06T00:00:00.000+00:00",
"lang": "en",
"value": "Vendor Notified"
},
{
"time": "2024-05-08T00:00:00.000+00:00",
"lang": "en",
"value": "Disclosed"
}
]
},
"adp": [
{
"title": "CISA ADP Vulnrichment",
"metrics": [
{
"other": {
"type": "ssvc",
"content": {
"id": "CVE-2024-4567",
"role": "CISA Coordinator",
"options": [
{
"Exploitation": "none"
},
{
"Automatable": "no"
},
{
"Technical Impact": "partial"
}
],
"version": "2.0.3",
"timestamp": "2024-05-11T16:56:12.695905Z"
}
}
}
],
"providerMetadata": {
"orgId": "134c704f-9b21-4f2e-91b3-4a467353bcc0",
"shortName": "CISA-ADP",
"dateUpdated": "2024-06-04T17:54:44.162Z"
}
},
{
"providerMetadata": {
"orgId": "af854a3a-2127-422b-91ae-364da2661108",
"shortName": "CVE",
"dateUpdated": "2024-08-01T20:47:40.724Z"
},
"title": "CVE Program Container",
"references": [
{
"url": "https://www.wordfence.com/threat-intel/vulnerabilities/id/c63ff9d7-6a14-4186-8550-4e5c50855e7f?source=cve",
"tags": [
"x_transferred"
]
},
{
"url": "https://plugins.trac.wordpress.org/changeset/3082885/themify-shortcodes",
"tags": [
"x_transferred"
]
}
]
}
]
}
}

View File

@@ -0,0 +1,19 @@
# CISA ICS Advisory Seed Data
This directory is reserved for **seed data** sourced from the community-maintained [ICS Advisory Project](https://github.com/icsadvprj/ICS-Advisory-Project). The project republishes CISA ICS advisories under the **Open Database License (ODbL) v1.0**. StellaOps uses these CSV snapshots to bootstrap offline environments before the official GovDelivery credentials arrive.
> ⚠️ **Licence notice** By downloading and using the CSV files you agree to the ODbL requirements (attribution, share-alike, and notice preservation). See [`LICENSE-ODBL.md`](https://github.com/icsadvprj/ICS-Advisory-Project/blob/main/LICENSE.md) for the full text.
## Usage
1. Run `scripts/fetch-ics-cisa-seed.sh` (or the PowerShell variant) to download the latest snapshots into this directory.
2. The files are ignored by Git to avoid committing third-party data; include them explicitly when building an Offline Update Kit.
3. When you later switch to live GovDelivery ingestion, keep the CSVs around as historical fixtures—do **not** treat them as an authoritative source once the live connector is enabled.
### Suggested Artefacts
- `CISA_ICS_ADV_Master.csv` cumulative advisory dataset (2010 → present)
- `CISA_ICS_ADV_<YYYY_MM_DD>.csv` point-in-time snapshots
- `ICSMA_CSV_<YYYY>.xlsx` medical device advisories (optional, sourced from the community mirror)
Keep the generated SHA-256 files alongside the CSVs so Offline Kit packaging can verify integrity.

View File

@@ -1,7 +1,7 @@
````markdown
# Feedser Vulnerability Conflict Resolution Rules
# Concelier Vulnerability Conflict Resolution Rules
This document defines the canonical, deterministic conflict resolution strategy for merging vulnerability data from **NVD**, **GHSA**, and **OSV** in Feedser.
This document defines the canonical, deterministic conflict resolution strategy for merging vulnerability data from **NVD**, **GHSA**, and **OSV** in Concelier.
---
@@ -128,7 +128,7 @@ return out
## 🧰 Optional C# Helper Class
`StellaOps.Feedser.Core/CanonicalMerger.cs`
`StellaOps.Concelier.Core/CanonicalMerger.cs`
Implements:

View File

@@ -1,11 +1,11 @@
<Project>
<PropertyGroup>
<FeedserPluginOutputRoot Condition="'$(FeedserPluginOutputRoot)' == ''">$(SolutionDir)PluginBinaries</FeedserPluginOutputRoot>
<FeedserPluginOutputRoot Condition="'$(FeedserPluginOutputRoot)' == '' and '$(SolutionDir)' == ''">$(MSBuildThisFileDirectory)PluginBinaries</FeedserPluginOutputRoot>
<ConcelierPluginOutputRoot Condition="'$(ConcelierPluginOutputRoot)' == ''">$(SolutionDir)PluginBinaries</ConcelierPluginOutputRoot>
<ConcelierPluginOutputRoot Condition="'$(ConcelierPluginOutputRoot)' == '' and '$(SolutionDir)' == ''">$(MSBuildThisFileDirectory)PluginBinaries</ConcelierPluginOutputRoot>
<AuthorityPluginOutputRoot Condition="'$(AuthorityPluginOutputRoot)' == ''">$(SolutionDir)PluginBinaries\Authority</AuthorityPluginOutputRoot>
<AuthorityPluginOutputRoot Condition="'$(AuthorityPluginOutputRoot)' == '' and '$(SolutionDir)' == ''">$(MSBuildThisFileDirectory)PluginBinaries\Authority</AuthorityPluginOutputRoot>
<IsFeedserPlugin Condition="'$(IsFeedserPlugin)' == '' and $([System.String]::Copy('$(MSBuildProjectName)').StartsWith('StellaOps.Feedser.Source.'))">true</IsFeedserPlugin>
<IsFeedserPlugin Condition="'$(IsFeedserPlugin)' == '' and $([System.String]::Copy('$(MSBuildProjectName)').StartsWith('StellaOps.Feedser.Exporter.'))">true</IsFeedserPlugin>
<IsConcelierPlugin Condition="'$(IsConcelierPlugin)' == '' and $([System.String]::Copy('$(MSBuildProjectName)').StartsWith('StellaOps.Concelier.Connector.'))">true</IsConcelierPlugin>
<IsConcelierPlugin Condition="'$(IsConcelierPlugin)' == '' and $([System.String]::Copy('$(MSBuildProjectName)').StartsWith('StellaOps.Concelier.Exporter.'))">true</IsConcelierPlugin>
<IsAuthorityPlugin Condition="'$(IsAuthorityPlugin)' == '' and $([System.String]::Copy('$(MSBuildProjectName)').StartsWith('StellaOps.Authority.Plugin.'))">true</IsAuthorityPlugin>
</PropertyGroup>
@@ -24,10 +24,10 @@
<PackageReference Include="xunit" Version="2.9.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
<PackageReference Include="Microsoft.Extensions.TimeProvider.Testing" Version="8.4.0" />
<Compile Include="$(MSBuildThisFileDirectory)StellaOps.Feedser.Tests.Shared\AssemblyInfo.cs" Link="Shared\AssemblyInfo.cs" />
<Compile Include="$(MSBuildThisFileDirectory)StellaOps.Feedser.Tests.Shared\MongoFixtureCollection.cs" Link="Shared\MongoFixtureCollection.cs" />
<ProjectReference Include="$(MSBuildThisFileDirectory)StellaOps.Feedser.Testing\StellaOps.Feedser.Testing.csproj" />
<Using Include="StellaOps.Feedser.Testing" />
<Compile Include="$(MSBuildThisFileDirectory)StellaOps.Concelier.Tests.Shared\AssemblyInfo.cs" Link="Shared\AssemblyInfo.cs" />
<Compile Include="$(MSBuildThisFileDirectory)StellaOps.Concelier.Tests.Shared\MongoFixtureCollection.cs" Link="Shared\MongoFixtureCollection.cs" />
<ProjectReference Include="$(MSBuildThisFileDirectory)StellaOps.Concelier.Testing\StellaOps.Concelier.Testing.csproj" />
<Using Include="StellaOps.Concelier.Testing" />
<Using Include="Xunit" />
</ItemGroup>
</Project>

View File

@@ -1,18 +1,18 @@
<Project>
<Target Name="FeedserCopyPluginArtifacts" AfterTargets="Build" Condition="'$(IsFeedserPlugin)' == 'true'">
<Target Name="ConcelierCopyPluginArtifacts" AfterTargets="Build" Condition="'$(IsConcelierPlugin)' == 'true'">
<PropertyGroup>
<FeedserPluginOutputDirectory>$(FeedserPluginOutputRoot)\$(MSBuildProjectName)</FeedserPluginOutputDirectory>
<ConcelierPluginOutputDirectory>$(ConcelierPluginOutputRoot)\$(MSBuildProjectName)</ConcelierPluginOutputDirectory>
</PropertyGroup>
<MakeDir Directories="$(FeedserPluginOutputDirectory)" />
<MakeDir Directories="$(ConcelierPluginOutputDirectory)" />
<ItemGroup>
<FeedserPluginArtifacts Include="$(TargetPath)" />
<FeedserPluginArtifacts Include="$(TargetPath).deps.json" Condition="Exists('$(TargetPath).deps.json')" />
<FeedserPluginArtifacts Include="$(TargetDir)$(TargetName).pdb" Condition="Exists('$(TargetDir)$(TargetName).pdb')" />
<ConcelierPluginArtifacts Include="$(TargetPath)" />
<ConcelierPluginArtifacts Include="$(TargetPath).deps.json" Condition="Exists('$(TargetPath).deps.json')" />
<ConcelierPluginArtifacts Include="$(TargetDir)$(TargetName).pdb" Condition="Exists('$(TargetDir)$(TargetName).pdb')" />
</ItemGroup>
<Copy SourceFiles="@(FeedserPluginArtifacts)" DestinationFolder="$(FeedserPluginOutputDirectory)" SkipUnchangedFiles="true" />
<Copy SourceFiles="@(ConcelierPluginArtifacts)" DestinationFolder="$(ConcelierPluginOutputDirectory)" SkipUnchangedFiles="true" />
</Target>
<Target Name="AuthorityCopyPluginArtifacts" AfterTargets="Build" Condition="'$(IsAuthorityPlugin)' == 'true'">

View File

@@ -80,6 +80,7 @@ Heres a quick, practical idea to make your version-range modeling cleaner and
* **Emit items, not a monolith**: have the builder return `IEnumerable<NormalizedVersionRule>`.
* **Normalize early**: resolve “aliases” (`1.2.x`, `^1.2.3`, distro styles) into canonical `(type,min,max,…)` before persistence.
* **Traceability**: include `notes`/`sourceRef` on each rule so you can re-materialize provenance during audits.
* **Lean projection helper**: when you only need normalized rules (and not the intermediate primitives), prefer `SemVerRangeRuleBuilder.BuildNormalizedRules(rawRange, patchedVersion, provenanceNote)` to skip manual projections.
### C# sketch

View File

@@ -12,15 +12,15 @@ public class StellaOpsPrincipalBuilderTests
public void NormalizedScopes_AreSortedDeduplicatedLowerCased()
{
var builder = new StellaOpsPrincipalBuilder()
.WithScopes(new[] { "Feedser.Jobs.Trigger", " feedser.jobs.trigger ", "AUTHORITY.USERS.MANAGE" })
.WithAudiences(new[] { " api://feedser ", "api://cli", "api://feedser" });
.WithScopes(new[] { "Concelier.Jobs.Trigger", " concelier.jobs.trigger ", "AUTHORITY.USERS.MANAGE" })
.WithAudiences(new[] { " api://concelier ", "api://cli", "api://concelier" });
Assert.Equal(
new[] { "authority.users.manage", "feedser.jobs.trigger" },
new[] { "authority.users.manage", "concelier.jobs.trigger" },
builder.NormalizedScopes);
Assert.Equal(
new[] { "api://cli", "api://feedser" },
new[] { "api://cli", "api://concelier" },
builder.Audiences);
}
@@ -38,8 +38,8 @@ public class StellaOpsPrincipalBuilderTests
.WithTokenId(Guid.NewGuid().ToString("N"))
.WithAuthenticationMethod("password")
.WithAuthenticationType(" custom ")
.WithScopes(new[] { "Feedser.Jobs.Trigger", "AUTHORITY.USERS.MANAGE" })
.WithAudience(" api://feedser ")
.WithScopes(new[] { "Concelier.Jobs.Trigger", "AUTHORITY.USERS.MANAGE" })
.WithAudience(" api://concelier ")
.WithIssuedAt(now)
.WithExpires(now.AddMinutes(5))
.AddClaim(" custom ", " value ");
@@ -57,13 +57,13 @@ public class StellaOpsPrincipalBuilderTests
Assert.Equal("value", principal.FindFirstValue("custom"));
var scopeClaims = principal.Claims.Where(claim => claim.Type == StellaOpsClaimTypes.ScopeItem).Select(claim => claim.Value).ToArray();
Assert.Equal(new[] { "authority.users.manage", "feedser.jobs.trigger" }, scopeClaims);
Assert.Equal(new[] { "authority.users.manage", "concelier.jobs.trigger" }, scopeClaims);
var scopeList = principal.FindFirstValue(StellaOpsClaimTypes.Scope);
Assert.Equal("authority.users.manage feedser.jobs.trigger", scopeList);
Assert.Equal("authority.users.manage concelier.jobs.trigger", scopeList);
var audienceClaims = principal.Claims.Where(claim => claim.Type == StellaOpsClaimTypes.Audience).Select(claim => claim.Value).ToArray();
Assert.Equal(new[] { "api://feedser" }, audienceClaims);
Assert.Equal(new[] { "api://concelier" }, audienceClaims);
var issuedAt = principal.FindFirstValue("iat");
Assert.Equal(now.ToUnixTimeSeconds().ToString(), issuedAt);

View File

@@ -37,7 +37,7 @@ public class StellaOpsProblemResultFactoryTests
public void InsufficientScope_AddsScopeExtensions()
{
var result = StellaOpsProblemResultFactory.InsufficientScope(
new[] { StellaOpsScopes.FeedserJobsTrigger },
new[] { StellaOpsScopes.ConcelierJobsTrigger },
new[] { StellaOpsScopes.AuthorityUsersManage },
instance: "/jobs/trigger");
@@ -46,7 +46,7 @@ public class StellaOpsProblemResultFactoryTests
var details = Assert.IsType<ProblemDetails>(result.ProblemDetails);
Assert.Equal("https://docs.stella-ops.org/problems/insufficient-scope", details.Type);
Assert.Equal("insufficient_scope", details.Extensions["error"]);
Assert.Equal(new[] { StellaOpsScopes.FeedserJobsTrigger }, Assert.IsType<string[]>(details.Extensions["required_scopes"]));
Assert.Equal(new[] { StellaOpsScopes.ConcelierJobsTrigger }, Assert.IsType<string[]>(details.Extensions["required_scopes"]));
Assert.Equal(new[] { StellaOpsScopes.AuthorityUsersManage }, Assert.IsType<string[]>(details.Extensions["granted_scopes"]));
Assert.Equal("/jobs/trigger", details.Instance);
}

View File

@@ -9,14 +9,14 @@ namespace StellaOps.Auth.Abstractions;
public static class StellaOpsScopes
{
/// <summary>
/// Scope required to trigger Feedser jobs.
/// Scope required to trigger Concelier jobs.
/// </summary>
public const string FeedserJobsTrigger = "feedser.jobs.trigger";
public const string ConcelierJobsTrigger = "concelier.jobs.trigger";
/// <summary>
/// Scope required to manage Feedser merge operations.
/// Scope required to manage Concelier merge operations.
/// </summary>
public const string FeedserMerge = "feedser.merge";
public const string ConcelierMerge = "concelier.merge";
/// <summary>
/// Scope granting administrative access to Authority user management.
@@ -40,8 +40,8 @@ public static class StellaOpsScopes
private static readonly HashSet<string> KnownScopes = new(StringComparer.OrdinalIgnoreCase)
{
FeedserJobsTrigger,
FeedserMerge,
ConcelierJobsTrigger,
ConcelierMerge,
AuthorityUsersManage,
AuthorityClientsManage,
AuthorityAuditRead,

View File

@@ -15,13 +15,13 @@ public class StellaOpsAuthClientOptionsTests
ClientId = "cli",
HttpTimeout = TimeSpan.FromSeconds(15)
};
options.DefaultScopes.Add(" Feedser.Jobs.Trigger ");
options.DefaultScopes.Add("feedser.jobs.trigger");
options.DefaultScopes.Add(" Concelier.Jobs.Trigger ");
options.DefaultScopes.Add("concelier.jobs.trigger");
options.DefaultScopes.Add("AUTHORITY.USERS.MANAGE");
options.Validate();
Assert.Equal(new[] { "authority.users.manage", "feedser.jobs.trigger" }, options.NormalizedScopes);
Assert.Equal(new[] { "authority.users.manage", "concelier.jobs.trigger" }, options.NormalizedScopes);
Assert.Equal(new Uri("https://authority.test"), options.AuthorityUri);
Assert.Equal<TimeSpan>(options.RetryDelays, options.NormalizedRetryDelays);
}

View File

@@ -21,7 +21,7 @@ public class StellaOpsTokenClientTests
var timeProvider = new FakeTimeProvider(DateTimeOffset.Parse("2025-02-01T00:00:00Z"));
var responses = new Queue<HttpResponseMessage>();
responses.Enqueue(CreateJsonResponse("{\"token_endpoint\":\"https://authority.test/connect/token\",\"jwks_uri\":\"https://authority.test/jwks\"}"));
responses.Enqueue(CreateJsonResponse("{\"access_token\":\"abc\",\"token_type\":\"Bearer\",\"expires_in\":120,\"scope\":\"feedser.jobs.trigger\"}"));
responses.Enqueue(CreateJsonResponse("{\"access_token\":\"abc\",\"token_type\":\"Bearer\",\"expires_in\":120,\"scope\":\"concelier.jobs.trigger\"}"));
responses.Enqueue(CreateJsonResponse("{\"keys\":[]}"));
var handler = new StubHttpMessageHandler((request, cancellationToken) =>
@@ -37,7 +37,7 @@ public class StellaOpsTokenClientTests
Authority = "https://authority.test",
ClientId = "cli"
};
options.DefaultScopes.Add("feedser.jobs.trigger");
options.DefaultScopes.Add("concelier.jobs.trigger");
options.Validate();
var optionsMonitor = new TestOptionsMonitor<StellaOpsAuthClientOptions>(options);
@@ -49,7 +49,7 @@ public class StellaOpsTokenClientTests
var result = await client.RequestPasswordTokenAsync("user", "pass");
Assert.Equal("abc", result.AccessToken);
Assert.Contains("feedser.jobs.trigger", result.Scopes);
Assert.Contains("concelier.jobs.trigger", result.Scopes);
await client.CacheTokenAsync("key", result.ToCacheEntry());
var cached = await client.GetCachedTokenAsync("key");

View File

@@ -19,8 +19,8 @@ public class ServiceCollectionExtensionsTests
.AddInMemoryCollection(new Dictionary<string, string?>
{
["Authority:ResourceServer:Authority"] = "https://authority.example",
["Authority:ResourceServer:Audiences:0"] = "api://feedser",
["Authority:ResourceServer:RequiredScopes:0"] = "feedser.jobs.trigger",
["Authority:ResourceServer:Audiences:0"] = "api://concelier",
["Authority:ResourceServer:RequiredScopes:0"] = "concelier.jobs.trigger",
["Authority:ResourceServer:BypassNetworks:0"] = "127.0.0.1/32"
})
.Build();
@@ -37,8 +37,8 @@ public class ServiceCollectionExtensionsTests
Assert.NotNull(jwtOptions.Authority);
Assert.Equal(new Uri("https://authority.example/"), new Uri(jwtOptions.Authority!));
Assert.True(jwtOptions.TokenValidationParameters.ValidateAudience);
Assert.Contains("api://feedser", jwtOptions.TokenValidationParameters.ValidAudiences);
Assert.Contains("api://concelier", jwtOptions.TokenValidationParameters.ValidAudiences);
Assert.Equal(TimeSpan.FromSeconds(60), jwtOptions.TokenValidationParameters.ClockSkew);
Assert.Equal(new[] { "feedser.jobs.trigger" }, resourceOptions.NormalizedScopes);
Assert.Equal(new[] { "concelier.jobs.trigger" }, resourceOptions.NormalizedScopes);
}
}

View File

@@ -17,12 +17,12 @@ public class StellaOpsResourceServerOptionsTests
TokenClockSkew = TimeSpan.FromSeconds(30)
};
options.Audiences.Add(" api://feedser ");
options.Audiences.Add("api://feedser");
options.Audiences.Add("api://feedser-admin");
options.Audiences.Add(" api://concelier ");
options.Audiences.Add("api://concelier");
options.Audiences.Add("api://concelier-admin");
options.RequiredScopes.Add(" Feedser.Jobs.Trigger ");
options.RequiredScopes.Add("feedser.jobs.trigger");
options.RequiredScopes.Add(" Concelier.Jobs.Trigger ");
options.RequiredScopes.Add("concelier.jobs.trigger");
options.RequiredScopes.Add("AUTHORITY.USERS.MANAGE");
options.BypassNetworks.Add("127.0.0.1/32");
@@ -32,8 +32,8 @@ public class StellaOpsResourceServerOptionsTests
options.Validate();
Assert.Equal(new Uri("https://authority.stella-ops.test"), options.AuthorityUri);
Assert.Equal(new[] { "api://feedser", "api://feedser-admin" }, options.Audiences);
Assert.Equal(new[] { "authority.users.manage", "feedser.jobs.trigger" }, options.NormalizedScopes);
Assert.Equal(new[] { "api://concelier", "api://concelier-admin" }, options.Audiences);
Assert.Equal(new[] { "authority.users.manage", "concelier.jobs.trigger" }, options.NormalizedScopes);
Assert.True(options.BypassMatcher.IsAllowed(IPAddress.Parse("127.0.0.1")));
Assert.True(options.BypassMatcher.IsAllowed(IPAddress.IPv6Loopback));
}

View File

@@ -24,10 +24,10 @@ public class StellaOpsScopeAuthorizationHandlerTests
});
var (handler, accessor) = CreateHandler(optionsMonitor, remoteAddress: IPAddress.Parse("10.0.0.1"));
var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.FeedserJobsTrigger });
var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.ConcelierJobsTrigger });
var principal = new StellaOpsPrincipalBuilder()
.WithSubject("user-1")
.WithScopes(new[] { StellaOpsScopes.FeedserJobsTrigger })
.WithScopes(new[] { StellaOpsScopes.ConcelierJobsTrigger })
.Build();
var context = new AuthorizationHandlerContext(new[] { requirement }, principal, accessor.HttpContext);
@@ -48,7 +48,7 @@ public class StellaOpsScopeAuthorizationHandlerTests
});
var (handler, accessor) = CreateHandler(optionsMonitor, remoteAddress: IPAddress.Parse("127.0.0.1"));
var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.FeedserJobsTrigger });
var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.ConcelierJobsTrigger });
var principal = new ClaimsPrincipal(new ClaimsIdentity());
var context = new AuthorizationHandlerContext(new[] { requirement }, principal, accessor.HttpContext);
@@ -67,7 +67,7 @@ public class StellaOpsScopeAuthorizationHandlerTests
});
var (handler, accessor) = CreateHandler(optionsMonitor, remoteAddress: IPAddress.Parse("203.0.113.10"));
var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.FeedserJobsTrigger });
var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.ConcelierJobsTrigger });
var principal = new ClaimsPrincipal(new ClaimsIdentity());
var context = new AuthorizationHandlerContext(new[] { requirement }, principal, accessor.HttpContext);

View File

@@ -4,6 +4,6 @@ ASP.NET Core helpers that enable resource servers to authenticate with **StellaO
- `AddStellaOpsResourceServerAuthentication` extension for JWT bearer + scope policies.
- Network bypass mask evaluation for on-host automation.
- Consistent `ProblemDetails` responses and policy helpers shared with Feedser/Backend services.
- Consistent `ProblemDetails` responses and policy helpers shared with Concelier/Backend services.
Pair this package with `StellaOps.Auth.Abstractions` and `StellaOps.Auth.Client` for end-to-end Authority integration.

View File

@@ -15,7 +15,8 @@ public class StandardClientProvisioningStoreTests
public async Task CreateOrUpdateAsync_HashesSecretAndPersistsDocument()
{
var store = new TrackingClientStore();
var provisioning = new StandardClientProvisioningStore("standard", store);
var revocations = new TrackingRevocationStore();
var provisioning = new StandardClientProvisioningStore("standard", store, revocations, TimeProvider.System);
var registration = new AuthorityClientRegistration(
clientId: "bootstrap-client",
@@ -63,4 +64,21 @@ public class StandardClientProvisioningStoreTests
return ValueTask.FromResult(removed);
}
}
private sealed class TrackingRevocationStore : IAuthorityRevocationStore
{
public List<AuthorityRevocationDocument> Upserts { get; } = new();
public ValueTask UpsertAsync(AuthorityRevocationDocument document, CancellationToken cancellationToken)
{
Upserts.Add(document);
return ValueTask.CompletedTask;
}
public ValueTask<bool> RemoveAsync(string category, string revocationId, CancellationToken cancellationToken)
=> ValueTask.FromResult(true);
public ValueTask<IReadOnlyList<AuthorityRevocationDocument>> GetActiveAsync(DateTimeOffset asOf, CancellationToken cancellationToken)
=> ValueTask.FromResult<IReadOnlyList<AuthorityRevocationDocument>>(Array.Empty<AuthorityRevocationDocument>());
}
}

View File

@@ -5,6 +5,7 @@ using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Options;
using Mongo2Go;
@@ -58,6 +59,21 @@ public class StandardPluginRegistrarTests
services.AddLogging();
services.AddSingleton<IMongoDatabase>(database);
services.AddSingleton<IAuthorityClientStore>(new InMemoryClientStore());
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
services.AddSingleton(TimeProvider.System);
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
services.AddSingleton(TimeProvider.System);
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
services.AddSingleton(TimeProvider.System);
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
services.AddSingleton(TimeProvider.System);
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
services.AddSingleton(TimeProvider.System);
services.AddSingleton(TimeProvider.System);
services.AddSingleton(TimeProvider.System);
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
var registrar = new StandardPluginRegistrar();
registrar.Register(new AuthorityPluginRegistrationContext(services, pluginContext, configuration));
@@ -83,6 +99,53 @@ public class StandardPluginRegistrarTests
Assert.True(verification.User?.RequiresPasswordReset);
}
[Fact]
public void Register_LogsWarning_WhenPasswordPolicyWeaker()
{
using var runner = MongoDbRunner.Start(singleNodeReplSet: true);
var client = new MongoClient(runner.ConnectionString);
var database = client.GetDatabase("registrar-password-policy");
var configuration = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary<string, string?>
{
["passwordPolicy:minimumLength"] = "6",
["passwordPolicy:requireUppercase"] = "false",
["passwordPolicy:requireLowercase"] = "false",
["passwordPolicy:requireDigit"] = "false",
["passwordPolicy:requireSymbol"] = "false"
})
.Build();
var manifest = new AuthorityPluginManifest(
"standard",
"standard",
true,
typeof(StandardPluginRegistrar).Assembly.GetName().Name,
typeof(StandardPluginRegistrar).Assembly.Location,
new[] { AuthorityPluginCapabilities.Password },
new Dictionary<string, string?>(),
"standard.yaml");
var pluginContext = new AuthorityPluginContext(manifest, configuration);
var services = new ServiceCollection();
var loggerProvider = new CapturingLoggerProvider();
services.AddLogging(builder => builder.AddProvider(loggerProvider));
services.AddSingleton<IMongoDatabase>(database);
services.AddSingleton<IAuthorityClientStore>(new InMemoryClientStore());
var registrar = new StandardPluginRegistrar();
registrar.Register(new AuthorityPluginRegistrationContext(services, pluginContext, configuration));
using var provider = services.BuildServiceProvider();
_ = provider.GetRequiredService<StandardUserCredentialStore>();
Assert.Contains(loggerProvider.Entries, entry =>
entry.Level == LogLevel.Warning &&
entry.Category.Contains(typeof(StandardPluginRegistrar).FullName!, StringComparison.Ordinal) &&
entry.Message.Contains("weaker password policy", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public void Register_ForcesPasswordCapability_WhenManifestMissing()
{
@@ -106,6 +169,8 @@ public class StandardPluginRegistrarTests
services.AddLogging();
services.AddSingleton<IMongoDatabase>(database);
services.AddSingleton<IAuthorityClientStore>(new InMemoryClientStore());
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
services.AddSingleton(TimeProvider.System);
var registrar = new StandardPluginRegistrar();
registrar.Register(new AuthorityPluginRegistrationContext(services, pluginContext, configuration));
@@ -209,6 +274,61 @@ public class StandardPluginRegistrarTests
}
}
internal sealed record CapturedLogEntry(string Category, LogLevel Level, string Message);
internal sealed class CapturingLoggerProvider : ILoggerProvider
{
public List<CapturedLogEntry> Entries { get; } = new();
public ILogger CreateLogger(string categoryName) => new CapturingLogger(categoryName, Entries);
public void Dispose()
{
}
private sealed class CapturingLogger : ILogger
{
private readonly string category;
private readonly List<CapturedLogEntry> entries;
public CapturingLogger(string category, List<CapturedLogEntry> entries)
{
this.category = category;
this.entries = entries;
}
public IDisposable BeginScope<TState>(TState state) where TState : notnull => NullScope.Instance;
public bool IsEnabled(LogLevel logLevel) => true;
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
{
entries.Add(new CapturedLogEntry(category, logLevel, formatter(state, exception)));
}
private sealed class NullScope : IDisposable
{
public static readonly NullScope Instance = new();
public void Dispose()
{
}
}
}
}
internal sealed class StubRevocationStore : IAuthorityRevocationStore
{
public ValueTask UpsertAsync(AuthorityRevocationDocument document, CancellationToken cancellationToken)
=> ValueTask.CompletedTask;
public ValueTask<bool> RemoveAsync(string category, string revocationId, CancellationToken cancellationToken)
=> ValueTask.FromResult(false);
public ValueTask<IReadOnlyList<AuthorityRevocationDocument>> GetActiveAsync(DateTimeOffset asOf, CancellationToken cancellationToken)
=> ValueTask.FromResult<IReadOnlyList<AuthorityRevocationDocument>>(Array.Empty<AuthorityRevocationDocument>());
}
internal sealed class InMemoryClientStore : IAuthorityClientStore
{
private readonly Dictionary<string, AuthorityClientDocument> clients = new(StringComparer.OrdinalIgnoreCase);

View File

@@ -71,6 +71,41 @@ internal sealed class PasswordPolicyOptions
throw new InvalidOperationException($"Standard plugin '{pluginName}' requires passwordPolicy.minimumLength to be greater than zero.");
}
}
public bool IsWeakerThan(PasswordPolicyOptions other)
{
if (other is null)
{
return false;
}
if (MinimumLength < other.MinimumLength)
{
return true;
}
if (!RequireUppercase && other.RequireUppercase)
{
return true;
}
if (!RequireLowercase && other.RequireLowercase)
{
return true;
}
if (!RequireDigit && other.RequireDigit)
{
return true;
}
if (!RequireSymbol && other.RequireSymbol)
{
return true;
}
return false;
}
}
internal sealed class LockoutOptions

Some files were not shown because too many files have changed in this diff Show More