diff --git a/.gitea/workflows/authority-key-rotation.yml b/.gitea/workflows/authority-key-rotation.yml new file mode 100644 index 00000000..01dec741 --- /dev/null +++ b/.gitea/workflows/authority-key-rotation.yml @@ -0,0 +1,163 @@ +# .gitea/workflows/authority-key-rotation.yml +# Manual workflow to push a new Authority signing key using OPS3 tooling + +name: Authority Key Rotation + +on: + workflow_dispatch: + inputs: + environment: + description: 'Target environment name (used to select secrets/vars)' + required: true + default: 'staging' + type: choice + options: + - staging + - production + authority_url: + description: 'Override Authority URL (leave blank to use env-specific secret)' + required: false + default: '' + type: string + key_id: + description: 'New signing key identifier (kid)' + required: true + type: string + key_path: + description: 'Path (as Authority sees it) to the PEM key' + required: true + type: string + source: + description: 'Signing key source loader (default: file)' + required: false + default: 'file' + type: string + algorithm: + description: 'Signing algorithm (default: ES256)' + required: false + default: 'ES256' + type: string + provider: + description: 'Preferred crypto provider hint' + required: false + default: '' + type: string + metadata: + description: 'Optional key=value metadata entries (comma-separated)' + required: false + default: '' + type: string + +jobs: + rotate: + runs-on: ubuntu-22.04 + environment: ${{ inputs.environment }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Resolve Authority configuration + id: config + run: | + set -euo pipefail + + env_name=${{ inputs.environment }} + echo "Environment: $env_name" + + bootstrap_key="" + authority_url="${{ inputs.authority_url }}" + + # Helper to prefer secrets over variables and fall back to shared defaults + resolve_var() { + local name="$1" + local default="$2" + local value="${{ secrets[name] }}" + if [ -z "$value" ]; then value="${{ vars[name] }}"; fi + if [ -z "$value" ]; then value="$default"; fi + printf '%s' "$value" + } + + key_name="${env_name^^}_AUTHORITY_BOOTSTRAP_KEY" + bootstrap_key="$(resolve_var "$key_name" "")" + if [ -z "$bootstrap_key" ]; then + bootstrap_key="$(resolve_var "AUTHORITY_BOOTSTRAP_KEY" "")" + fi + + if [ -z "$bootstrap_key" ]; then + echo "::error::Missing bootstrap key secret (expected $key_name or AUTHORITY_BOOTSTRAP_KEY)" + exit 1 + fi + + if [ -z "$authority_url" ]; then + url_name="${env_name^^}_AUTHORITY_URL" + authority_url="$(resolve_var "$url_name" "")" + if [ -z "$authority_url" ]; then + authority_url="$(resolve_var "AUTHORITY_URL" "")" + fi + fi + + if [ -z "$authority_url" ]; then + echo "::error::Authority URL not provided and no secret/var found" + exit 1 + fi + + key_file="${RUNNER_TEMP}/authority-bootstrap-key" + printf '%s\n' "$bootstrap_key" > "$key_file" + chmod 600 "$key_file" + + echo "bootstrap-key-file=$key_file" >> "$GITHUB_OUTPUT" + echo "authority-url=$authority_url" >> "$GITHUB_OUTPUT" + + - name: Execute key rotation + id: rotate + shell: bash + env: + AUTHORITY_BOOTSTRAP_KEY_FILE: ${{ steps.config.outputs['bootstrap-key-file'] }} + AUTHORITY_URL: ${{ steps.config.outputs['authority-url'] }} + KEY_ID: ${{ inputs.key_id }} + KEY_PATH: ${{ inputs.key_path }} + KEY_SOURCE: ${{ inputs.source }} + KEY_ALGORITHM: ${{ inputs.algorithm }} + KEY_PROVIDER: ${{ inputs.provider }} + KEY_METADATA: ${{ inputs.metadata }} + run: | + set -euo pipefail + + bootstrap_key=$(cat "$AUTHORITY_BOOTSTRAP_KEY_FILE") + + metadata_args=() + if [ -n "$KEY_METADATA" ]; then + IFS=',' read -ra META <<< "$KEY_METADATA" + for entry in "${META[@]}"; do + trimmed="$(echo "$entry" | xargs)" + [ -z "$trimmed" ] && continue + metadata_args+=(-m "$trimmed") + done + fi + + provider_args=() + if [ -n "$KEY_PROVIDER" ]; then + provider_args+=(--provider "$KEY_PROVIDER") + fi + + ./ops/authority/key-rotation.sh \ + --authority-url "$AUTHORITY_URL" \ + --api-key "$bootstrap_key" \ + --key-id "$KEY_ID" \ + --key-path "$KEY_PATH" \ + --source "$KEY_SOURCE" \ + --algorithm "$KEY_ALGORITHM" \ + "${provider_args[@]}" \ + "${metadata_args[@]}" + + - name: JWKS summary + run: | + echo "✅ Rotation complete" + echo "Environment: ${{ inputs.environment }}" + echo "Authority: ${{ steps.config.outputs['authority-url'] }}" + echo "Key ID: ${{ inputs.key_id }}" + echo "Key Path: ${{ inputs.key_path }}" + echo "Source: ${{ inputs.source }}" + echo "Algorithm: ${{ inputs.algorithm }}" diff --git a/AGENTS.md b/AGENTS.md index 5a50c0f9..1b530c01 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,7 +119,10 @@ You main characteristics: - **Scoping**: Use each module’s `AGENTS.md` and `TASKS.md` to plan; autonomous agents must read `src/AGENTS.md` and the module docs before acting. - **Determinism**: Sort keys, normalize timestamps to UTC ISO‑8601, avoid non‑deterministic data in exports and tests. - **Status tracking**: Update your module’s `TASKS.md` as you progress (TODO → DOING → DONE/BLOCKED). Before starting of actual work - ensure you have set the task to DOING. When complete or stop update the status in corresponding TASKS.md or in ./SPRINTS.md file. +- **Coordination**: In case task is discovered as blocked on other team or task, according TASKS.md files that dependency is on needs to be changed by adding new tasks describing the requirement. the current task must be updated as completed. In case task changes, scope or requirements or rules - other documentations needs be updated accordingly. +- **Sprint synchronization**: When given task seek for relevant directory to work on from SPRINTS.md. Confirm its state on both SPRINTS.md and the relevant TODOS.md file. Always check the AGENTS.md in the relevant TODOS.md directory. - **Tests**: Add/extend fixtures and unit tests per change; never regress determinism or precedence. - **Test layout**: Use module-specific projects in `StellaOps.Feedser..Tests`; shared fixtures/harnesses live in `StellaOps.Feedser.Testing`. +- **Execution autonomous**: In case you need to continue with more than one options just continue sequentially, unless the continue requires design decision. --- diff --git a/SPRINTS.md b/SPRINTS.md index 80513e53..c511c84e 100644 --- a/SPRINTS.md +++ b/SPRINTS.md @@ -1,18 +1,18 @@ | Sprint | Theme | Tasks File Path | Status | Type of Specialist | Task ID | Task Description | | --- | --- | --- | --- | --- | --- | --- | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Models/TASKS.md | — | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-001 | SemVer primitive range-style metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Feedser.Models/AGENTS.md. This task lays the groundwork—complete the SemVer helper updates before teammates pick up FEEDMODELS-SCHEMA-01-002/003 and FEEDMODELS-SCHEMA-02-900. Use ./src/FASTER_MODELING_AND_NORMALIZATION.md for the target rule structure. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-001 | SemVer primitive range-style metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Feedser.Models/AGENTS.md. This task lays the groundwork—complete the SemVer helper updates before teammates pick up FEEDMODELS-SCHEMA-01-002/003 and FEEDMODELS-SCHEMA-02-900. Use ./src/FASTER_MODELING_AND_NORMALIZATION.md for the target rule structure. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-002 | Provenance decision rationale field
Instructions to work:
AdvisoryProvenance now carries `decisionReason` and docs/tests were updated. Connectors and merge tasks should populate the field when applying precedence/freshness/tie-breaker logic; see src/StellaOps.Feedser.Models/PROVENANCE_GUIDELINES.md for usage guidance. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-003 | Normalized version rules collection
Instructions to work:
`AffectedPackage.NormalizedVersions` and supporting comparer/docs/tests shipped. Connector owners must emit rule arrays per ./src/FASTER_MODELING_AND_NORMALIZATION.md and report progress via FEEDMERGE-COORD-02-900 so merge/storage backfills can proceed. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Models/TASKS.md | — | Team Models & Merge Leads | FEEDMODELS-SCHEMA-02-900 | Range primitives for SemVer/EVR/NEVRA metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Feedser.Models/AGENTS.md before resuming this stalled effort. Confirm helpers align with the new `NormalizedVersions` representation so connectors finishing in Sprint 2 can emit consistent metadata. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-02-900 | Range primitives for SemVer/EVR/NEVRA metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Feedser.Models/AGENTS.md before resuming this stalled effort. Confirm helpers align with the new `NormalizedVersions` representation so connectors finishing in Sprint 2 can emit consistent metadata. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Normalization/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter
Shared `SemVerRangeRuleBuilder` now outputs primitives + normalized rules per `FASTER_MODELING_AND_NORMALIZATION.md`; CVE/GHSA connectors consuming the API have verified fixtures. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill
AdvisoryStore dual-writes flattened `normalizedVersions` when `feedser.storage.enableSemVerStyle` is set; migration `20251011-semver-style-backfill` updates historical records and docs outline the rollout. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence
Storage now persists `provenance.decisionReason` for advisories and merge events; tests cover round-trips. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing
Bootstrapper seeds compound/sparse indexes for flattened normalized rules and `docs/dev/mongo_indices.md` documents query guidance. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor
Updated constructors/tests keep storage suites passing with the new feature flag defaults. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options
WebService now binds `authority.resilience.*` into `AddStellaOpsAuthClient`; integration tests verify retry/offline tolerance wiring. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Quickstart/install docs ship `authority.resilience.*` defaults, online vs air-gapped tuning, and monitoring references aligned with WebService coverage. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Guides now include the structured audit log fields (`route/status/subject/clientId/scopes/bypass/remote`) and SIEM alert guidance. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Feedser operator guide for enforcement cutoff
Operator guide details the 2025-12-31 cutoff checklist and env var rollout to disable `allowAnonymousFallback`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options
WebService wires `authority.resilience.*` into `AddStellaOpsAuthClient` and adds binding coverage via `AuthorityClientResilienceOptionsAreBound`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Install/runbooks document connected vs air-gapped resilience profiles and monitoring hooks. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Operator guides now call out `route/status/subject/clientId/scopes/bypass/remote` audit fields and SIEM triggers. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Feedser operator guide for enforcement cutoff
Install guide reiterates the 2025-12-31 cutoff and links audit signals to the rollout checklist. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | SEC3.HOST | Rate limiter policy binding
Authority host now applies configuration-driven fixed windows to `/token`, `/authorize`, and `/internal/*`; integration tests assert 429 + `Retry-After` headers; docs/config samples refreshed for Docs guild diagrams. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | SEC3.BUILD | Authority rate-limiter follow-through
`Security.RateLimiting` now fronts token/authorize/internal limiters; Authority + Configuration matrices (`dotnet test src/StellaOps.Authority/StellaOps.Authority.sln`, `dotnet test src/StellaOps.Configuration.Tests/StellaOps.Configuration.Tests.csproj`) passed on 2025-10-11; awaiting #authority-core broadcast. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | PLG6.DOC | Plugin developer guide polish
Section 9 now documents rate limiter metadata, config keys, and lockout interplay; YAML samples updated alongside Authority config templates. | @@ -20,30 +20,30 @@ | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-002 | VINCE note detail fetcher
Summary planner queues VINCE note detail endpoints, persists raw JSON with SHA/ETag metadata, and records retry/backoff metrics. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-003 | DTO & parser implementation
Added VINCE DTO aggregate, Markdown→text sanitizer, vendor/status/vulnerability parsers, and parser regression fixture. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-004 | Canonical mapping & range primitives
VINCE DTO aggregate flows through `CertCcMapper`, emitting vendor range primitives + normalized version rules that persist via `_advisoryStore`. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DOING (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-005 | Deterministic fixtures/tests
Fetch→parse→map integration suite in place; snapshot harness reactivation pending fixture regeneration & documentation of the `UPDATE_CERTCC_FIXTURES` flow. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-006 | Telemetry & documentation
WebService now exports the `StellaOps.Feedser.Source.CertCc` meter and fetch integration tests assert `certcc.*` counters; README updated with observability guidance. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DOING (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-007 | Connector test harness remediation
Canned-response harness restored; snapshot fixtures still being realigned and regeneration steps documented. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-005 | Deterministic fixtures/tests
Snapshot harness refreshed 2025-10-12; `certcc-*.snapshot.json` regenerated and regression suite green without UPDATE flag drift. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-006 | Telemetry & documentation
`CertCcDiagnostics` publishes summary/detail/parse/map metrics (meter `StellaOps.Feedser.Source.CertCc`), README documents instruments, and log guidance captured for Ops on 2025-10-12. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-007 | Connector test harness remediation
Harness now wires `AddSourceCommon`, resets `FakeTimeProvider`, and passes canned-response regression run dated 2025-10-12. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | BLOCKED (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-008 | Snapshot coverage handoff
Upstream repo version lacks SemVer primitives + provenance decision reason fields, so snapshot regeneration fails; resume once Models/Storage sprint lands those changes. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | TODO | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-012 | Schema sync & snapshot regen follow-up
Re-run fixture updates and deliver Merge handoff after SemVer style + decision reason updates merge into main. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-012 | Schema sync & snapshot regen follow-up
Fixtures regenerated with normalizedVersions + provenance decision reasons; handoff notes updated for Merge backfill 2025-10-12. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-009 | Detail/map reintegration plan
Staged reintegration plan published in `src/StellaOps.Feedser.Source.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md`; coordinates enablement with FEEDCONN-CERTCC-02-004. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | TODO (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-010 | Partial-detail graceful degradation
Ensure missing VINCE endpoints downgrade gracefully without fatal errors; add coverage for partial note ingestion paths. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-010 | Partial-detail graceful degradation
Detail fetch now tolerates 404/403/410 responses and regression tests cover mixed endpoint availability. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Distro.RedHat/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-REDHAT-02-001 | Fixture validation sweep
Instructions to work:
Fixtures regenerated post-model-helper rollout; provenance ordering and normalizedVersions scaffolding verified via tests. Conflict resolver deltas logged in src/StellaOps.Feedser.Source.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md for Sprint 3 consumers. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | — | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-001 | Canonical mapping & range primitives | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-001 | Canonical mapping & range primitives
Mapper emits SemVer rules (`scheme=apple:*`); fixtures regenerated with trimmed references + new RSR coverage, update tooling finalized. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-002 | Deterministic fixtures/tests
Sanitized live fixtures + regression snapshots wired into tests; normalized rule coverage asserted. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-003 | Telemetry & documentation
Apple meter metrics wired into Feedser WebService OpenTelemetry configuration; README and fixtures document normalizedVersions coverage. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-004 | Live HTML regression sweep
Live support.apple.com sweep captured iOS/macOS/RSR cases with deterministic sanitizers. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-004 | Live HTML regression sweep
Sanitised HT125326/HT125328/HT106355/HT214108/HT215500 fixtures recorded and regression tests green on 2025-10-12. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-005 | Fixture regeneration tooling
`UPDATE_APPLE_FIXTURES=1` flow fetches & rewrites fixtures; README documents usage.
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Feedser.Source.Vndr.Apple/AGENTS.md. Resume stalled tasks, ensuring normalizedVersions output and fixtures align with ./src/FASTER_MODELING_AND_NORMALIZATION.md before handing data to the conflict sprint. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | — | Team Connector Normalized Versions Rollout | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance
Team instructions: Read ./AGENTS.md and each module's AGENTS file. Adopt the `NormalizedVersions` array emitted by the models sprint, wiring provenance `decisionReason` where merge overrides occur. Follow ./src/FASTER_MODELING_AND_NORMALIZATION.md; report via src/StellaOps.Feedser.Merge/TASKS.md (FEEDMERGE-COORD-02-900). Progress 2025-10-11: GHSA/OSV emit normalized arrays with refreshed fixtures; CVE mapper now surfaces SemVer normalized ranges; NVD/KEV adoption pending; outstanding follow-ups include FEEDSTORAGE-DATA-02-001, FEEDMERGE-ENGINE-02-002, and rolling `tools/FixtureUpdater` updates across connectors. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Osv/TASKS.md | — | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Nvd/TASKS.md | — | Team Connector Normalized Versions Rollout | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Cve/TASKS.md | — | Team Connector Normalized Versions Rollout | FEEDCONN-CVE-02-003 | CVE normalized versions uplift | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Kev/TASKS.md | — | Team Connector Normalized Versions Rollout | FEEDCONN-KEV-02-003 | KEV normalized versions propagation | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Osv/TASKS.md | — | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-04-003 | OSV parity fixture refresh | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance
Team instructions: Read ./AGENTS.md and each module's AGENTS file. Adopt the `NormalizedVersions` array emitted by the models sprint, wiring provenance `decisionReason` where merge overrides occur. Follow ./src/FASTER_MODELING_AND_NORMALIZATION.md; report via src/StellaOps.Feedser.Merge/TASKS.md (FEEDMERGE-COORD-02-900). Progress 2025-10-11: GHSA/OSV emit normalized arrays with refreshed fixtures; CVE mapper now surfaces SemVer normalized ranges; NVD/KEV adoption pending; outstanding follow-ups include FEEDSTORAGE-DATA-02-001, FEEDMERGE-ENGINE-02-002, and rolling `tools/FixtureUpdater` updates across connectors. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Cve/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-CVE-02-003 | CVE normalized versions uplift | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Kev/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-KEV-02-003 | KEV normalized versions propagation | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-04-003 | OSV parity fixture refresh | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-10) | Team WebService & Authority | FEEDWEB-DOCS-01-001 | Document authority toggle & scope requirements
Quickstart carries toggle/scope guidance pending docs guild review (no change this sprint). | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options
WebService binds `authority.resilience.*` into `AddStellaOpsAuthClient`; integration tests validate retry/offline wiring. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Operator docs cover resilience defaults, environment-specific tuning, and monitoring cues. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Audit logging examples now document bypass fields and recommended alerts. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Feedser operator guide for enforcement cutoff
Enforcement checklist and env overrides documented ahead of the 2025-12-31 cutoff. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options
WebService wires `authority.resilience.*` into `AddStellaOpsAuthClient` and adds binding coverage via `AuthorityClientResilienceOptionsAreBound`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Operator docs now outline connected vs air-gapped resilience profiles and monitoring cues. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Audit logging guidance highlights `route/status/subject/clientId/scopes/bypass/remote` fields and SIEM alerts. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Feedser operator guide for enforcement cutoff
Install guide reiterates the 2025-12-31 cutoff and ties audit signals to rollout checks. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-OPS-01-006 | Rename plugin drop directory to namespaced path
Build outputs, tests, and docs now target `StellaOps.Feedser.PluginBinaries`/`StellaOps.Authority.PluginBinaries`. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-OPS-01-007 | Authority resilience adoption
Deployment docs and CLI notes explain the LIB5 resilience knobs for rollout.
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Feedser.WebService/AGENTS.md. These items were mid-flight; resume implementation ensuring docs/operators receive timely updates. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHCORE-ENGINE-01-001 | CORE8.RL — Rate limiter plumbing validated; integration tests green and docs handoff recorded for middleware ordering + Retry-After headers (see `docs/dev/authority-rate-limit-tuning-outline.md` for continuing guidance). | @@ -58,14 +58,14 @@ | Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-004 | GHSA credits & ecosystem severity mapping | | Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | TODO | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-005 | GitHub quota monitoring & retries | | Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | TODO | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-006 | Production credential & scheduler rollout | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | TODO | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-007 | Credit parity regression fixtures | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-007 | Credit parity regression fixtures | | Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps | | Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-004 | NVD CVSS & CWE precedence payloads | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Nvd/TASKS.md | TODO | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-005 | NVD merge/export parity regression | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-005 | NVD merge/export parity regression | | Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness | | Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-004 | OSV references & credits alignment | | Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Osv/TASKS.md | TODO | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-005 | Fixture updater workflow
Instructions to work:
Read ./AGENTS.md and respective module AGENTS. Implement builder integration, provenance, and supporting docs using ./src/FASTER_MODELING_AND_NORMALIZATION.md and ensure outputs satisfy the precedence matrix in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Acsc/TASKS.md | Research DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-ACSC-02-001 … 02-008 | RSS endpoints catalogued 2025-10-11, HTTP/2 error + pagination validation and client compatibility task opened. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Acsc/TASKS.md | Implementation DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-ACSC-02-001 … 02-008 | Fetch pipeline, DTO parser, canonical mapper, fixtures, and README shipped 2025-10-12; downstream export integration still pending future tasks. | | Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Cccs/TASKS.md | Research DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CCCS-02-001 … 02-007 | Atom feed verified 2025-10-11, history/caching review and FR locale enumeration pending. | | Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.CertBund/TASKS.md | Research DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CERTBUND-02-001 … 02-007 | BSI RSS directory confirmed CERT-Bund feed 2025-10-11, history assessment pending. | | Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Kisa/TASKS.md | Research DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-KISA-02-001 … 02-007 | KNVD RSS endpoint identified 2025-10-11, access headers/session strategy outstanding. | @@ -88,8 +88,8 @@ | Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-002 | Override metrics instrumentation
Merge events capture per-field decisions; counters/logs align with conflict rules. | | Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-003 | Reference & credit union pipeline
Canonical merge preserves unions with updated tests. | | Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-QA-04-001 | End-to-end conflict regression suite
Added regression tests (`AdvisoryMergeServiceTests`) covering canonical + precedence flow.
Instructions to work:
Read ./AGENTS.md and merge AGENTS. Integrate the canonical merger, instrument metrics, and deliver comprehensive regression tests following ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | — | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-002 | GHSA conflict regression fixtures | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Source.Nvd/TASKS.md | — | Team Connector Regression Fixtures | FEEDCONN-NVD-04-002 | NVD conflict regression fixtures | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Source.Osv/TASKS.md | — | Team Connector Regression Fixtures | FEEDCONN-OSV-04-002 | OSV conflict regression fixtures
Instructions to work:
Read ./AGENTS.md and module AGENTS. Produce fixture triples supporting the precedence/tie-breaker paths defined in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md and hand them to Merge QA. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-002 | GHSA conflict regression fixtures | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Source.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-NVD-04-002 | NVD conflict regression fixtures | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Source.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-OSV-04-002 | OSV conflict regression fixtures
Instructions to work:
Read ./AGENTS.md and module AGENTS. Produce fixture triples supporting the precedence/tie-breaker paths defined in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md and hand them to Merge QA. | | Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | DONE (2025-10-11) | Team Documentation Guild – Conflict Guidance | FEEDDOCS-DOCS-05-001 | Feedser Conflict Rules
Runbook published at `docs/ops/feedser-conflict-resolution.md`; metrics/log guidance aligned with Sprint 3 merge counters. | -| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | TODO | Team Documentation Guild – Conflict Guidance | FEEDDOCS-DOCS-05-002 | Conflict runbook ops rollout
Instructions to work:
Read ./AGENTS.md and docs/AGENTS.md. Socialise the conflict runbook with Feedser Ops, tune alert thresholds, and record change-log linkage once sign-off is captured. Use ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md for ongoing rule references. | +| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | TODO | Team Documentation Guild – Conflict Guidance | FEEDDOCS-DOCS-05-002 | Conflict runbook ops rollout
Instructions to work:
Read ./AGENTS.md and docs/AGENTS.md. Once GHSA/NVD/OSV regression fixtures (FEEDCONN-GHSA-04-002, FEEDCONN-NVD-04-002, FEEDCONN-OSV-04-002) are delivered, schedule the Ops review, apply the alert thresholds captured in `docs/ops/feedser-authority-audit-runbook.md`, and record change-log linkage after sign-off. Use ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md for ongoing rule references. | diff --git a/TODOS.md b/TODOS.md index 86403496..fc6f4d21 100644 --- a/TODOS.md +++ b/TODOS.md @@ -2,8 +2,8 @@ | Task | Status | Notes | |---|---|---| |FEEDCONN-CERTCC-02-005 Deterministic fixtures/tests|DONE (2025-10-11)|Snapshot regression for summary/detail fetch landed; fixtures regenerate via `UPDATE_CERTCC_FIXTURES`.| -|FEEDCONN-CERTCC-02-008 Snapshot coverage handoff|DONE (2025-10-11)|`CertCcConnectorSnapshotTests` produce documents/state/request snapshots and document the refresh workflow.| -|FEEDCONN-CERTCC-02-007 Connector test harness remediation|TODO|Need to restore Source.CertCc harness (`AddSourceCommon`, canned responses) so parser regression can run in CI.| +|FEEDCONN-CERTCC-02-008 Snapshot coverage handoff|DONE (2025-10-11)|Fixtures + README guidance shipped; QA can rerun with `UPDATE_CERTCC_FIXTURES=1` and share recorded-request diff with Merge.| +|FEEDCONN-CERTCC-02-007 Connector test harness remediation|DONE (2025-10-11)|Harness now resets time provider, wires Source.Common, and verifies VINCE canned responses across fetch→parse→map.| |FEEDCONN-CERTCC-02-009 Detail/map reintegration plan|DONE (2025-10-11)|Plan published in `src/StellaOps.Feedser.Source.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md`; outlines staged enablement + rollback.| # Connector Apple Status diff --git a/certificates/authority-signing-2025-dev.pem b/certificates/authority-signing-2025-dev.pem new file mode 100644 index 00000000..106113ab --- /dev/null +++ b/certificates/authority-signing-2025-dev.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIIX2ZUujxnKwidwmPeUlhYKafkxno39luXI6700/hv0roAoGCCqGSM49 +AwEHoUQDQgAEvliBfYvF+aKLX25ZClPwqYt6xdTQ9aP9fbEVTW8xQb61alaa8Tae +bjIvg4IFlD+0zzv7ciLVFuYhNkY+UkVnZg== +-----END EC PRIVATE KEY----- diff --git a/dep_tmp.txt b/dep_tmp.txt new file mode 100644 index 00000000..e69de29b diff --git a/docs/09_API_CLI_REFERENCE.md b/docs/09_API_CLI_REFERENCE.md index 483e2310..511fc10e 100755 --- a/docs/09_API_CLI_REFERENCE.md +++ b/docs/09_API_CLI_REFERENCE.md @@ -228,9 +228,11 @@ See `docs/dev/32_AUTH_CLIENT_GUIDE.md` for recommended profiles (online vs. air- | `stellaops-cli db merge` | Run canonical merge reconcile | — | Calls `POST /jobs/merge:reconcile`; exit code `0` on acceptance, `1` on failures/conflicts | | `stellaops-cli db export` | Kick JSON / Trivy exports | `--format ` (default `json`)
`--delta`
`--publish-full/--publish-delta`
`--bundle-full/--bundle-delta` | Sets `{ delta = true }` parameter when requested and can override ORAS/bundle toggles per run | | `stellaops-cli auth ` | Manage cached tokens for StellaOps Authority | `auth login --force` (ignore cache)
`auth status`
`auth whoami` | Uses `StellaOps.Auth.Client`; honours `StellaOps:Authority:*` configuration, stores tokens under `~/.stellaops/tokens` by default, and `whoami` prints subject/scope/expiry | +| `stellaops-cli auth revoke export` | Export the Authority revocation bundle | `--output ` (defaults to CWD) | Writes `revocation-bundle.json`, `.json.jws`, and `.json.sha256`; verifies the digest locally and includes key metadata in the log summary. | +| `stellaops-cli auth revoke verify` | Validate a revocation bundle offline | `--bundle ` `--signature ` `--key `
`--verbose` | Verifies detached JWS signatures, reports the computed SHA-256, and can fall back to cached JWKS when `--key` is omitted. | +| `stellaops-cli config show` | Display resolved configuration | — | Masks secret values; helpful for air‑gapped installs | When running on an interactive terminal without explicit override flags, the CLI uses Spectre.Console prompts to let you choose per-run ORAS/offline bundle behaviour. -| `stellaops-cli config show` | Display resolved configuration | — | Masks secret values; helpful for air‑gapped installs | **Logging & exit codes** @@ -340,6 +342,30 @@ Drop `appsettings.local.json` or `.yaml` beside the binary to override per envir --- +### 2.6 Authority Admin APIs + +Administrative endpoints live under `/internal/*` on the Authority host and require the bootstrap API key (`x-stellaops-bootstrap-key`). Responses are deterministic and audited via `AuthEventRecord`. + +| Path | Method | Description | +| ---- | ------ | ----------- | +| `/internal/revocations/export` | GET | Returns the revocation bundle (JSON + detached JWS + digest). Mirrors the output of `stellaops-cli auth revoke export`. | +| `/internal/signing/rotate` | POST | Promotes a new signing key and marks the previous key as retired without restarting the service. | + +**Rotate request body** + +```json +{ + "keyId": "authority-signing-2025", + "location": "../certificates/authority-signing-2025.pem", + "source": "file", + "provider": "default" +} +``` + +The API responds with the active `kid`, previous key (if any), and the set of retired key identifiers. Always export a fresh revocation bundle after rotation so downstream mirrors receive signatures from the new key. + +--- + ## 3 First‑Party CLI Tools ### 3.1 `stella` diff --git a/docs/11_AUTHORITY.md b/docs/11_AUTHORITY.md new file mode 100644 index 00000000..faa6d946 --- /dev/null +++ b/docs/11_AUTHORITY.md @@ -0,0 +1,161 @@ +# StellaOps Authority Service + +> **Status:** Drafted 2025-10-12 (CORE5B.DOC / DOC1.AUTH) – aligns with Authority revocation store, JWKS rotation, and bootstrap endpoints delivered in Sprint 1. + +## 1. Purpose +The **StellaOps Authority** service issues OAuth2/OIDC tokens for every StellaOps module (Feedser, Backend, Agent, Zastava) and exposes the policy controls required in sovereign/offline environments. Authority is built as a minimal ASP.NET host that: + +- brokers password, client-credentials, and device-code flows through pluggable identity providers; +- persists access/refresh/device tokens in MongoDB with deterministic schemas for replay analysis and air-gapped audit copies; +- distributes revocation bundles and JWKS material so downstream services can enforce lockouts without direct database access; +- offers bootstrap APIs for first-run provisioning and key rotation without redeploying binaries. + +Authority is deployed alongside Feedser in air-gapped environments and never requires outbound internet access. All trusted metadata (OpenIddict discovery, JWKS, revocation bundles) is cacheable, signed, and reproducible. + +## 2. Component Architecture +Authority is composed of five cooperating subsystems: + +1. **Minimal API host** – configures OpenIddict endpoints (`/token`, `/authorize`, `/revoke`, `/jwks`) and structured logging/telemetry. Rate limiting hooks (`AuthorityRateLimiter`) wrap every request. +2. **Plugin host** – loads `StellaOps.Authority.Plugin.*.dll` assemblies, applies capability metadata, and exposes password/client provisioning surfaces through dependency injection. +3. **Mongo storage** – persists tokens, revocations, bootstrap invites, and plugin state in deterministic collections indexed for offline sync (`authority_tokens`, `authority_revocations`, etc.). +4. **Cryptography layer** – `StellaOps.Cryptography` abstractions manage password hashing, signing keys, JWKS export, and detached JWS generation. +5. **Offline ops APIs** – internal endpoints under `/internal/*` provide administrative flows (bootstrap users/clients, revocation export) guarded by API keys and deterministic audit events. + +A high-level sequence for password logins: + +``` +Client -> /token (password grant) + -> Rate limiter & audit hooks + -> Plugin credential store (Argon2id verification) + -> Token persistence (Mongo authority_tokens) + -> Response (access/refresh tokens + deterministic claims) +``` + +## 3. Token Lifecycle & Persistence +Authority persists every issued token in MongoDB so operators can audit or revoke without scanning distributed caches. + +- **Collection:** `authority_tokens` +- **Key fields:** + - `tokenId`, `type` (`access_token`, `refresh_token`, `device_code`, `authorization_code`) + - `subjectId`, `clientId`, ordered `scope` array + - `status` (`valid`, `revoked`, `expired`), `createdAt`, optional `expiresAt` + - `revokedAt`, machine-readable `revokedReason`, optional `revokedReasonDescription` + - `revokedMetadata` (string dictionary for plugin-specific context) +- **Persistence flow:** `PersistTokensHandler` stamps missing JWT IDs, normalises scopes, and stores every principal emitted by OpenIddict. +- **Revocation flow:** `AuthorityTokenStore.UpdateStatusAsync` flips status, records the reason metadata, and is invoked by token revocation handlers and plugin provisioning events (e.g., disabling a user). +- **Expiry maintenance:** `AuthorityTokenStore.DeleteExpiredAsync` prunes non-revoked tokens past their `expiresAt` timestamp. Operators should schedule this in maintenance windows if large volumes of tokens are issued. + +### Expectations for resource servers +Resource servers (Feedser WebService, Backend, Agent) **must not** assume in-memory caches are authoritative. They should: + +- cache `/jwks` and `/revocations/export` responses within configured lifetimes; +- honour `revokedReason` metadata when shaping audit trails; +- treat `status != "valid"` or missing tokens as immediate denial conditions. + +## 4. Revocation Pipeline +Authority centralises revocation in `authority_revocations` with deterministic categories: + +| Category | Meaning | Required fields | +| --- | --- | --- | +| `token` | Specific OAuth token revoked early. | `revocationId` (token id), `tokenType`, optional `clientId`, `subjectId` | +| `subject` | All tokens for a subject disabled. | `revocationId` (= subject id) | +| `client` | OAuth client registration revoked. | `revocationId` (= client id) | +| `key` | Signing/JWE key withdrawn. | `revocationId` (= key id) | + +`RevocationBundleBuilder` flattens Mongo documents into canonical JSON, sorts entries by (`category`, `revocationId`, `revokedAt`), and signs exports using detached JWS (RFC 7797) with cosign-compatible headers. + +**Export surfaces** (deterministic output, suitable for Offline Kit): + +- CLI: `stella auth revoke export --output ./out` writes `revocation-bundle.json`, `.jws`, `.sha256`. +- API: `GET /internal/revocations/export` (requires bootstrap API key) returns the same payload. +- Verification: `stella auth revoke verify` validates schema, digest, and detached JWS using cached JWKS or offline keys. + +**Consumer guidance:** + +1. Mirror `revocation-bundle.json*` alongside Feedser exports. Offline agents fetch both over the existing update channel. +2. Use bundle `sequence` and `bundleId` to detect replay or monotonicity regressions. Ignore bundles with older sequence numbers unless `bundleId` changes and `issuedAt` advances. +3. Treat `revokedReason` taxonomy as machine-friendly codes (`compromised`, `rotation`, `policy`, `lifecycle`). Translating to human-readable logs is the consumer’s responsibility. + +## 5. Signing Keys & JWKS Rotation +Authority signs revocation bundles and publishes JWKS entries via the new signing manager: + +- **Configuration (`authority.yaml`):** + ```yaml + signing: + enabled: true + algorithm: ES256 # Defaults to ES256 + keySource: file # Loader identifier (file, vault, etc.) + provider: default # Optional preferred crypto provider + activeKeyId: authority-signing-dev + keyPath: "../certificates/authority-signing-dev.pem" + additionalKeys: + - keyId: authority-signing-dev-2024 + path: "../certificates/authority-signing-dev-2024.pem" + source: "file" + ``` +- **Sources:** The default loader supports PEM files relative to the content root; additional loaders can be registered via `IAuthoritySigningKeySource`. +- **Providers:** Keys are registered against the `ICryptoProviderRegistry`, so alternative implementations (HSM, libsodium) can be plugged in without changing host code. +- **JWKS output:** `GET /jwks` lists every signing key with `status` metadata (`active`, `retired`). Old keys remain until operators remove them from configuration, allowing verification of historical bundles/tokens. + +### Rotation SOP (no downtime) +1. Generate a new P-256 private key (PEM) on an offline workstation and place it where the Authority host can read it (e.g., `../certificates/authority-signing-2025.pem`). +2. Call the authenticated admin API: + ```bash + curl -sS -X POST https://authority.example.com/internal/signing/rotate \ + -H "x-stellaops-bootstrap-key: ${BOOTSTRAP_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "keyId": "authority-signing-2025", + "location": "../certificates/authority-signing-2025.pem", + "source": "file" + }' + ``` +3. Verify the response reports the previous key as retired and fetch `/jwks` to confirm the new `kid` appears with `status: "active"`. +4. Persist the old key path in `signing.additionalKeys` (the rotation API updates in-memory options; rewrite the YAML to match so restarts remain consistent). +5. If you prefer automation, trigger the `.gitea/workflows/authority-key-rotation.yml` workflow with the new `keyId`/`keyPath`; it wraps `ops/authority/key-rotation.sh` and reads environment-specific secrets. The older key will be marked `retired` and appended to `signing.additionalKeys`. +6. Re-run `stella auth revoke export` so revocation bundles are signed with the new key. Downstream caches should refresh JWKS within their configured lifetime (`StellaOpsAuthorityOptions.Signing` + client cache tolerance). + +The rotation API leverages the same cryptography abstractions as revocation signing; no restart is required and the previous key is marked `retired` but kept available for verification. + +## 6. Bootstrap & Administrative Endpoints +Administrative APIs live under `/internal/*` and require the bootstrap API key plus rate-limiter compliance. + +| Endpoint | Method | Description | +| --- | --- | --- | +| `/internal/users` | `POST` | Provision initial administrative accounts through the registered password-capable plug-in. Emits structured audit events. | +| `/internal/clients` | `POST` | Provision OAuth clients (client credentials / device code). | +| `/internal/revocations/export` | `GET` | Export revocation bundle + detached JWS + digest. | +| `/internal/signing/rotate` | `POST` | Promote a new signing key (see SOP above). Request body accepts `keyId`, `location`, optional `source`, `algorithm`, `provider`, and metadata. | + +All administrative calls emit `AuthEventRecord` entries enriched with correlation IDs, PII tags, and network metadata for offline SOC ingestion. + +## 7. Configuration Reference + +| Section | Key | Description | Notes | +| --- | --- | --- | --- | +| Root | `issuer` | Absolute HTTPS issuer advertised to clients. | Required. Loopback HTTP allowed only for development. | +| Tokens | `accessTokenLifetime`, `refreshTokenLifetime`, etc. | Lifetimes for each grant (access, refresh, device, authorization code, identity). | Enforced during issuance; persisted on each token document. | +| Storage | `storage.connectionString` | MongoDB connection string. | Required even for tests; offline kits ship snapshots for seeding. | +| Signing | `signing.enabled` | Enable JWKS/revocation signing. | Disable only for development. | +| Signing | `signing.algorithm` | Signing algorithm identifier. | Currently ES256; additional curves can be wired through crypto providers. | +| Signing | `signing.keySource` | Loader identifier (`file`, `vault`, custom). | Determines which `IAuthoritySigningKeySource` resolves keys. | +| Signing | `signing.keyPath` | Relative/absolute path understood by the loader. | Stored as-is; rotation request should keep it in sync with filesystem layout. | +| Signing | `signing.activeKeyId` | Active JWKS / revocation signing key id. | Exposed as `kid` in JWKS and bundles. | +| Signing | `signing.additionalKeys[].keyId` | Retired key identifier retained for verification. | Manager updates this automatically after rotation; keep YAML aligned. | +| Signing | `signing.additionalKeys[].source` | Loader identifier per retired key. | Defaults to `signing.keySource` if omitted. | +| Security | `security.rateLimiting` | Fixed-window limits for `/token`, `/authorize`, `/internal/*`. | See `docs/security/rate-limits.md` for tuning. | +| Bootstrap | `bootstrap.apiKey` | Shared secret required for `/internal/*`. | Only required when `bootstrap.enabled` is true. | + +## 8. Offline & Sovereign Operation +- **No outbound dependencies:** Authority only contacts MongoDB and local plugins. Discovery and JWKS are cached by clients with offline tolerances (`AllowOfflineCacheFallback`, `OfflineCacheTolerance`). Operators should mirror these responses for air-gapped use. +- **Structured logging:** Every revocation export, signing rotation, bootstrap action, and token issuance emits structured logs with `traceId`, `client_id`, `subjectId`, and `network.remoteIp` where applicable. Mirror logs to your SIEM to retain audit trails without central connectivity. +- **Determinism:** Sorting rules in token and revocation exports guarantee byte-for-byte identical artefacts given the same datastore state. Hashes and signatures remain stable across machines. + +## 9. Operational Checklist +- [ ] Protect the bootstrap API key and disable bootstrap endpoints (`bootstrap.enabled: false`) once initial setup is complete. +- [ ] Schedule `stella auth revoke export` (or `/internal/revocations/export`) at the same cadence as Feedser exports so bundles remain in lockstep. +- [ ] Rotate signing keys before expiration; keep at least one retired key until all cached bundles/tokens signed with it have expired. +- [ ] Monitor `/health` and `/ready` plus rate-limiter metrics to detect plugin outages early. +- [ ] Ensure downstream services cache JWKS and revocation bundles within tolerances; stale caches risk accepting revoked tokens. + +For plug-in specific requirements, refer to **[Authority Plug-in Developer Guide](dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md)**. For revocation bundle validation workflow, see **[Authority Revocation Bundle](security/revocation-bundle.md)**. diff --git a/docs/17_SECURITY_HARDENING_GUIDE.md b/docs/17_SECURITY_HARDENING_GUIDE.md index 92d33a10..9b95eaa6 100755 --- a/docs/17_SECURITY_HARDENING_GUIDE.md +++ b/docs/17_SECURITY_HARDENING_GUIDE.md @@ -159,6 +159,7 @@ cosign verify ghcr.io/stellaops/backend@sha256: \ showing whether a network bypass CIDR allowed the request. Configure your SIEM to alert when unauthenticated requests (`status=401`) appear with `bypass=true`, or when unexpected scopes invoke job triggers. + Detailed monitoring and response guidance lives in `docs/ops/feedser-authority-audit-runbook.md`. ##  8 Update & patch strategy diff --git a/docs/21_INSTALL_GUIDE.md b/docs/21_INSTALL_GUIDE.md index 310e44f4..0a363545 100755 --- a/docs/21_INSTALL_GUIDE.md +++ b/docs/21_INSTALL_GUIDE.md @@ -93,12 +93,20 @@ The Feedser container reads configuration from `etc/feedser.yaml` plus FEEDSER_AUTHORITY__CLIENTSECRETFILE="/run/secrets/feedser_authority_client" FEEDSER_AUTHORITY__BYPASSNETWORKS__0="127.0.0.1/32" FEEDSER_AUTHORITY__BYPASSNETWORKS__1="::1/128" + FEEDSER_AUTHORITY__RESILIENCE__ENABLERETRIES=true + FEEDSER_AUTHORITY__RESILIENCE__RETRYDELAYS__0="00:00:01" + FEEDSER_AUTHORITY__RESILIENCE__RETRYDELAYS__1="00:00:02" + FEEDSER_AUTHORITY__RESILIENCE__RETRYDELAYS__2="00:00:05" + FEEDSER_AUTHORITY__RESILIENCE__ALLOWOFFLINECACHEFALLBACK=true + FEEDSER_AUTHORITY__RESILIENCE__OFFLINECACHETOLERANCE="00:10:00" ``` Store the client secret outside source control (Docker secrets, mounted file, or Kubernetes Secret). Feedser loads the secret during post-configuration, so the value never needs to appear in the YAML template. + Connected sites can keep the retry ladder short (1 s, 2 s, 5 s) so job triggers fail fast when Authority is down. For air‑gapped or intermittently connected deployments, extend `RESILIENCE__OFFLINECACHETOLERANCE` (e.g. `00:30:00`) so cached discovery/JWKS data remains valid while the Offline Kit synchronises upstream changes. + 2. Redeploy Feedser: ```bash @@ -106,9 +114,10 @@ The Feedser container reads configuration from `etc/feedser.yaml` plus ``` 3. Tail the logs: `docker compose logs -f feedser`. Successful `/jobs*` calls now - emit `Feedser.Authorization.Audit` entries listing subject, client ID, scopes, - remote IP, and whether the bypass CIDR allowed the call. 401 denials always log - `bypassAllowed=false` so unauthenticated cron jobs are easy to catch. + emit `Feedser.Authorization.Audit` entries with `route`, `status`, `subject`, + `clientId`, `scopes`, `bypass`, and `remote` fields. 401 denials keep the same + shape—watch for `bypass=True`, which indicates a bypass CIDR accepted an anonymous + call. See `docs/ops/feedser-authority-audit-runbook.md` for a full audit/alerting checklist. > **Enforcement deadline** – keep `FEEDSER_AUTHORITY__ALLOWANONYMOUSFALLBACK=true` > only while validating the rollout. Set it to `false` (and restart Feedser) diff --git a/docs/README.md b/docs/README.md index c1a12ece..653c9b7b 100755 --- a/docs/README.md +++ b/docs/README.md @@ -40,11 +40,13 @@ Everything here is open‑source and versioned — when you check out a git ta - [`zastava_scanner.md`](08_MODULE_SPECIFICATIONS/zastava_scanner.md) - [`registry_scanner.md`](08_MODULE_SPECIFICATIONS/registry_scanner.md) - [`nightly_scheduler.md`](08_MODULE_SPECIFICATIONS/nightly_scheduler.md) -- **09 – [API & CLI Reference](09_API_CLI_REFERENCE.md)** -- **10 – [Plug‑in SDK Guide](10_PLUGIN_SDK_GUIDE.md)** -- **11 – [Data Schemas](11_DATA_SCHEMAS.md)** -- **12 – [Performance Workbook](12_PERFORMANCE_WORKBOOK.md)** -- **13 – [Release‑Engineering Playbook](13_RELEASE_ENGINEERING_PLAYBOOK.md)** +- **09 – [API & CLI Reference](09_API_CLI_REFERENCE.md)** +- **10 – [Plug‑in SDK Guide](10_PLUGIN_SDK_GUIDE.md)** +- **11 – [Authority Service](11_AUTHORITY.md)** +- **11 – [Data Schemas](11_DATA_SCHEMAS.md)** +- **12 – [Performance Workbook](12_PERFORMANCE_WORKBOOK.md)** +- **13 – [Release‑Engineering Playbook](13_RELEASE_ENGINEERING_PLAYBOOK.md)** +- **30 – [Fixture Maintenance](dev/fixtures.md)** ### User & operator guides - **14 – [Glossary](14_GLOSSARY_OF_TERMS.md)** @@ -53,9 +55,11 @@ Everything here is open‑source and versioned — when you check out a git ta - **18 – [Coding Standards](18_CODING_STANDARDS.md)** - **19 – [Test‑Suite Overview](19_TEST_SUITE_OVERVIEW.md)** - **21 – [Install Guide](21_INSTALL_GUIDE.md)** -- **22 – [CI/CD Recipes Library](ci/20_CI_RECIPES.md)** -- **23 – [FAQ](23_FAQ_MATRIX.md)** -- **24 – [Offline Update Kit Admin Guide](24_OUK_ADMIN_GUIDE.md)** +- **22 – [CI/CD Recipes Library](ci/20_CI_RECIPES.md)** +- **23 – [FAQ](23_FAQ_MATRIX.md)** +- **24 – [Offline Update Kit Admin Guide](24_OUK_ADMIN_GUIDE.md)** +- **26 – [Authority Key Rotation Playbook](ops/authority-key-rotation.md)** +- **25 – [Feedser Apple Connector Operations](ops/feedser-apple-operations.md)** ### Legal & licence - **29 – [Legal & Quota FAQ](29_LEGAL_FAQ_QUOTA.md)** diff --git a/docs/TASKS.md b/docs/TASKS.md index 9ac95b47..b3e8b7a7 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -3,10 +3,10 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| | DOC4.AUTH-PDG | REVIEW | Docs Guild, Plugin Team | PLG6.DOC | Copy-edit `docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md`, export lifecycle diagram, add LDAP RFC cross-link. | ✅ PR merged with polish; ✅ Diagram committed; ✅ Slack handoff posted. | -| DOC1.AUTH | TODO | Docs Guild, Authority Core | CORE5B.DOC | Draft `docs/11_AUTHORITY.md` covering architecture, configuration, bootstrap flows. | ✅ Architecture + config sections approved by Core; ✅ Samples reference latest options; ✅ Offline note added. | -| DOC3.Feedser-Authority | DOING (2025-10-10) | Docs Guild, DevEx | FSR4 | Polish operator/runbook sections (DOC3/DOC5) to document Feedser authority rollout, bypass logging, and enforcement checklist. | ✅ DOC3/DOC5 updated; ✅ enforcement deadline highlighted; ✅ Docs guild sign-off. | -| DOC5.Feedser-Runbook | TODO | Docs Guild | DOC3.Feedser-Authority | Produce dedicated Feedser authority audit runbook covering log fields, monitoring recommendations, and troubleshooting steps. | ✅ Runbook published; ✅ linked from DOC3/DOC5; ✅ alerting guidance included. | +| DOC1.AUTH | DONE (2025-10-12) | Docs Guild, Authority Core | CORE5B.DOC | Draft `docs/11_AUTHORITY.md` covering architecture, configuration, bootstrap flows. | ✅ Architecture + config sections approved by Core; ✅ Samples reference latest options; ✅ Offline note added. | +| DOC3.Feedser-Authority | DONE (2025-10-12) | Docs Guild, DevEx | FSR4 | Polish operator/runbook sections (DOC3/DOC5) to document Feedser authority rollout, bypass logging, and enforcement checklist. | ✅ DOC3/DOC5 updated with audit runbook references; ✅ enforcement deadline highlighted; ✅ Docs guild sign-off. | +| DOC5.Feedser-Runbook | DONE (2025-10-12) | Docs Guild | DOC3.Feedser-Authority | Produce dedicated Feedser authority audit runbook covering log fields, monitoring recommendations, and troubleshooting steps. | ✅ Runbook published; ✅ linked from DOC3/DOC5; ✅ alerting guidance included. | | FEEDDOCS-DOCS-05-001 | DONE (2025-10-11) | Docs Guild | FEEDMERGE-ENGINE-04-001, FEEDMERGE-ENGINE-04-002 | Publish Feedser conflict resolution runbook covering precedence workflow, merge-event auditing, and Sprint 3 metrics. | ✅ `docs/ops/feedser-conflict-resolution.md` committed; ✅ metrics/log tables align with latest merge code; ✅ Ops alert guidance handed to Feedser team. | -| FEEDDOCS-DOCS-05-002 | TODO | Docs Guild, Feedser Ops | FEEDDOCS-DOCS-05-001 | Capture ops sign-off: circulate conflict runbook, tune alert thresholds, and document rollout decisions in change log. | ✅ Ops review recorded; ✅ alert thresholds finalised; ✅ change-log entry linked from runbook. | +| FEEDDOCS-DOCS-05-002 | TODO | Docs Guild, Feedser Ops | FEEDDOCS-DOCS-05-001 | Capture ops sign-off: circulate conflict runbook, tune alert thresholds, and document rollout decisions in change log. | ✅ Ops review recorded; ✅ alert thresholds finalised using `docs/ops/feedser-authority-audit-runbook.md`; ✅ change-log entry linked from runbook once GHSA/NVD/OSV regression fixtures land. | > Update statuses (TODO/DOING/REVIEW/DONE/BLOCKED) as progress changes. Keep guides in sync with configuration samples under `etc/`. diff --git a/docs/assets/authority/authority-plugin-lifecycle.mmd b/docs/assets/authority/authority-plugin-lifecycle.mmd new file mode 100644 index 00000000..1aaa6565 --- /dev/null +++ b/docs/assets/authority/authority-plugin-lifecycle.mmd @@ -0,0 +1,17 @@ +%% Authority plug-in lifecycle sequence diagram (Mermaid) +flowchart LR + manifest[[Plugin Manifest
etc/authority.plugins/*.yaml]] + loader[AuthorityPluginConfigurationLoader
binds and validates options] + scanner[PluginHost Assembly Scan
StellaOps.Authority.Plugin.*] + registrar[IAuthorityPluginRegistrar
registers services & health checks] + runtime[Identity Provider Plugin
IIdentityProviderPlugin surface] + capabilities{Capability Metadata
password/mfa/bootstrap/clientProvisioning} + storage[(Credential Store
Mongo collections or custom backend)] + telemetry[[Structured Logs & Metrics
authority.*]] + + manifest --> loader --> scanner --> registrar --> runtime --> storage + scanner --> capabilities + capabilities --> runtime + runtime --> telemetry + loader -. emits deterministic config hashes .-> telemetry + storage -. readiness probes .-> runtime diff --git a/docs/assets/authority/authority-plugin-lifecycle.svg b/docs/assets/authority/authority-plugin-lifecycle.svg new file mode 100644 index 00000000..bd9d2a1c --- /dev/null +++ b/docs/assets/authority/authority-plugin-lifecycle.svg @@ -0,0 +1,91 @@ + + Authority plug-in lifecycle: manifest to runtime + + + + + + + + Authority plug-in lifecycle + + + + + Manifest YAML + etc/authority.plugins/*.yaml + Deterministic hashes + + + + + AuthorityPluginConfigurationLoader + binds + validates options + logs config fingerprints + + + + + PluginHost assembly scan + StellaOps.Authority.Plugin.* + loads IAuthorityPluginRegistrar + + + + + IAuthorityPluginRegistrar + register services & health checks + publish capability metadata + + + + + Identity Provider plug-in + IIdentityProviderPlugin + ready/readiness probes + + + + + + Capability metadata broadcast + password / mfa / bootstrap / clientProvisioning + + + + + Credential & audit storage + Mongo collections or custom backend + queried in readiness probes + + + + + Telemetry output + logs + metrics with correlation IDs + + + + + + + + + + + + + 1. Configuration + 2. Assembly discovery + 3. Registrar execution + 4. Runtime surface + diff --git a/docs/assets/authority/authority-rate-limit-flow.mmd b/docs/assets/authority/authority-rate-limit-flow.mmd new file mode 100644 index 00000000..fcae2e14 --- /dev/null +++ b/docs/assets/authority/authority-rate-limit-flow.mmd @@ -0,0 +1,27 @@ +%% Rate limit and lockout interplay for Standard plug-in (Mermaid) +sequenceDiagram + autonumber + participant Client as Client/App + participant Host as Authority Host + participant Limiter as Rate Limiter Middleware + participant Plugin as Standard Plugin + participant Store as Credential Store / Lockout State + + Client->>Host: POST /token (client_id, credentials) + Host->>Limiter: Check quota (client_id + remote_ip) + alt quota exceeded + Limiter-->>Host: Reject (429, retryAfter) + Host-->>Client: 429 Too Many Requests\nRetry-After header with limiter tags + else quota ok + Limiter-->>Host: Allow (remaining tokens) + Host->>Plugin: VerifyCredentials(subject) + Plugin->>Store: Load hashed password + lockout counters + Store-->>Plugin: Credential result + deterministic counter + alt lockout threshold reached + Plugin-->>Host: Locked (retryAfter=lockoutWindow) + Host-->>Client: 423 Locked\nRetry-After header + `authority.lockout` tag + else valid credentials + Plugin-->>Host: Success (issue tokens) + Host-->>Client: 200 OK + tokens + limiter metadata + end + end diff --git a/docs/assets/authority/authority-rate-limit-flow.svg b/docs/assets/authority/authority-rate-limit-flow.svg new file mode 100644 index 00000000..f2db1be5 --- /dev/null +++ b/docs/assets/authority/authority-rate-limit-flow.svg @@ -0,0 +1,105 @@ + + Authority rate limit and lockout flow + + + + + + + + + + + + + + + Client / App + + Authority Host + + Rate Limiter + + Standard Plug-in + + Credential Store + + + + + POST /token request + client_id + subject creds + + + + + Authority middleware + enriches context tags + + + + + Rate limiter window + client_id + IP keyed + + + + + Quota exceeded? + emit Retry-After & tags + + + + + Quota OK + pass remaining tokens + + + + + Verify credentials + hash compare + audit tags + + + + + Load lockout state + deterministic counters + + + + + Lockout threshold hit? + follow dedup precedence + + + + + Issue tokens or errors + include limiter metadata + + + + + + + + + + + + + + + + 429 path → add `authority.client_id`, `authority.remote_ip` tags for dashboards. + Lockout path → reuse precedence strategy from Feedser dedup (see DEDUP_CONFLICTS_RESOLUTION_ALGO.md). + diff --git a/docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md b/docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md index 36a4e76b..1b0ceaa4 100644 --- a/docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md +++ b/docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md @@ -1,6 +1,6 @@ # Authority Plug-in Developer Guide -> **Status:** Ready for Docs/DOC4 editorial review as of 2025-10-10. Content aligns with PLG6 acceptance criteria and references stable Authority primitives. +> **Status:** Updated 2025-10-11 (AUTHPLUG-DOCS-01-001) with lifecycle + limiter diagrams and refreshed rate-limit guidance aligned to PLG6 acceptance criteria. ## 1. Overview Authority plug-ins extend the **StellaOps Authority** service with custom identity providers, credential stores, and client-management logic. Unlike Feedser plug-ins (which ingest or export advisories), Authority plug-ins participate directly in authentication flows: @@ -17,6 +17,10 @@ Authority hosts follow a deterministic plug-in lifecycle. The flow below can be 3. **Registrar execution** – each assembly is searched for `IAuthorityPluginRegistrar` implementations. Registrars bind options, register services, and optionally queue bootstrap tasks. 4. **Runtime** – the host resolves `IIdentityProviderPlugin` instances, uses capability metadata to decide which OAuth grants to expose, and invokes health checks for readiness endpoints. +![Authority plug-in lifecycle diagram](../assets/authority/authority-plugin-lifecycle.svg) + +_Source:_ `docs/assets/authority/authority-plugin-lifecycle.mmd` + **Data persistence primer:** the standard Mongo-backed plugin stores users in collections named `authority_users_` and lockout metadata in embedded documents. Additional plugins must document their storage layout and provide deterministic collection naming to honour the Offline Kit replication process. ## 3. Capability Metadata @@ -100,7 +104,7 @@ Capability flags let the host reason about what your plug-in supports: - Optional `IClientProvisioningStore` for machine-to-machine clients. - `AuthorityIdentityProviderCapabilities` to advertise supported flows. - Password guidance: - - Prefer Argon2 (Security Guild upcoming recommendation); Standard plug-in currently ships PBKDF2 with easy swap via `IPasswordHasher`. + - Standard plug-in hashes via `ICryptoProvider` using Argon2id by default and emits PHC-compliant strings. Successful PBKDF2 logins trigger automatic rehashes so migrations complete gradually. See `docs/security/password-hashing.md` for tuning advice. - Enforce password policies before hashing to avoid storing weak credentials. - Health checks should probe backing stores (e.g., Mongo `ping`) and return `AuthorityPluginHealthResult` so `/ready` can surface issues. - When supporting additional factors (e.g., TOTP), implement `SupportsMfa` and document the enrolment flow for resource servers. @@ -111,14 +115,64 @@ Capability flags let the host reason about what your plug-in supports: - Never store raw secrets in git: allow operators to supply them via `.local.yaml`, environment variables, or injected secret files. Document which keys are mandatory. - Validate configuration as soon as the registrar runs; use explicit error messages to guide operators. The Standard plug-in now enforces complete bootstrap credentials (username + password) and positive lockout windows via `StandardPluginOptions.Validate`. - Cross-reference bootstrap workflows with `docs/ops/authority_bootstrap.md` (to be published alongside CORE6) so operators can reuse the same payload formats for manual provisioning. +- `passwordHashing` inherits defaults from `authority.security.passwordHashing`. Override only when hardware constraints differ per plug-in: + ```yaml + passwordHashing: + algorithm: Argon2id + memorySizeInKib: 19456 + iterations: 2 + parallelism: 1 + ``` + Invalid values (≤0) fail fast during startup, and legacy PBKDF2 hashes rehash automatically once the new algorithm succeeds. -## 8. Logging, Metrics, and Diagnostics +### 7.1 Token Persistence Contract +- The host automatically persists every issued principal (access, refresh, device, authorization code) in `authority_tokens`. Plug-in code **must not** bypass this store; use the provided `IAuthorityTokenStore` helpers when implementing custom flows. +- When a plug-in disables a subject or client outside the standard handlers, call `IAuthorityTokenStore.UpdateStatusAsync(...)` for each affected token so revocation bundles stay consistent. +- Supply machine-friendly `revokedReason` codes (`compromised`, `rotation`, `policy`, `lifecycle`, etc.) and optional `revokedMetadata` entries when invalidating credentials. These flow straight into `revocation-bundle.json` and should remain deterministic. +- Token scopes should be normalised (trimmed, unique, ordinal sort) before returning from plug-in verification paths. `TokenPersistenceHandlers` will keep that ordering for downstream consumers. + +### 7.2 Claims & Enrichment Checklist +- Authority always sets the OpenID Connect basics: `sub`, `client_id`, `preferred_username`, optional `name`, and `role` (for password flows). Plug-ins must use `IClaimsEnricher` to append additional claims in a **deterministic** order (sort arrays, normalise casing) so resource servers can rely on stable shapes. +- Recommended enrichment keys: + - `stellaops.realm` – plug-in/tenant identifier so services can scope policies. + - `stellaops.subject.type` – values such as `human`, `service`, `bootstrap`. + - `groups` / `projects` – sorted arrays describing operator entitlements. +- Claims visible in tokens should mirror what `/token` and `/userinfo` emit. Avoid injecting sensitive PII directly; mark values with `ClassifiedString.Personal` inside the plug-in so audit sinks can tag them appropriately. +- For client-credential flows, remember to enrich both the client principal and the validation path (`TokenValidationHandlers`) so refresh flows keep the same metadata. + +### 7.3 Revocation Bundles & Reasons +- Use `IAuthorityRevocationStore` to record subject/client/token revocations when credentials are deleted or rotated. Stick to the standard categories (`token`, `subject`, `client`, `key`). +- Include a deterministic `reason` string and optional `reasonDescription` so operators understand *why* a subject was revoked when inspecting bundles offline. +- Plug-ins should populate `metadata` with stable keys (e.g., `revokedBy`, `sourcePlugin`, `ticketId`) to simplify SOC correlation. The keys must be lowercase, ASCII, and free of secrets—bundles are mirrored to air-gapped agents. + +## 8. Rate Limiting & Lockout Interplay +Rate limiting and account lockouts are complementary controls. Plug-ins must surface both deterministically so operators can correlate limiter hits with credential rejections. + +**Baseline quotas** (from `docs/dev/authority-rate-limit-tuning-outline.md`): + +| Endpoint | Default policy | Notes | +|----------|----------------|-------| +| `/token` | 30 requests / 60s, queue 0 | Drop to 10/60s for untrusted ranges; raise only with WAF + monitoring. | +| `/authorize` | 60 requests / 60s, queue 10 | Reduce carefully; interactive UX depends on headroom. | +| `/internal/*` | Disabled by default; recommended 5/60s when enabled | Keep queue 0 for bootstrap APIs. | + +**Retry metadata:** The middleware stamps `Retry-After` plus tags `authority.client_id`, `authority.remote_ip`, and `authority.endpoint`. Plug-ins should keep these tags intact when crafting responses or telemetry so dashboards remain consistent. + +**Lockout counters:** Treat lockouts as **subject-scoped** decisions. When multiple instances update counters, reuse the deterministic tie-breakers documented in `src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md` (freshness overrides, precedence, and stable hashes) to avoid divergent lockout states across replicas. + +**Alerting hooks:** Emit structured logs/metrics when either the limiter or credential store rejects access. Suggested gauges include `aspnetcore_rate_limiting_rejections_total{limiter="authority-token"}` and any custom `auth.plugins..lockouts_total` counter. + +![Authority rate limit and lockout flow](../assets/authority/authority-rate-limit-flow.svg) + +_Source:_ `docs/assets/authority/authority-rate-limit-flow.mmd` + +## 9. Logging, Metrics, and Diagnostics - Always log via the injected `ILogger`; include `pluginName` and correlation IDs where available. - Activity/metric names should align with `AuthorityTelemetry` constants (`service.name=stellaops-authority`). - Expose additional diagnostics via structured logging rather than writing custom HTTP endpoints; the host will integrate these into `/health` and `/ready`. - Emit metrics with stable names (`auth.plugins..*`) when introducing custom instrumentation; coordinate with the Observability guild to reserve prefixes. -## 9. Testing & Tooling +## 10. Testing & Tooling - Unit tests: use Mongo2Go (or similar) to exercise credential stores without hitting production infrastructure (`StandardUserCredentialStoreTests` is a template). - Determinism: fix timestamps to UTC and sort outputs consistently; avoid random GUIDs unless stable. - Smoke tests: launch `dotnet run --project src/StellaOps.Authority/StellaOps.Authority` with your plug-in under `PluginBinaries/Authority` and verify `/ready`. @@ -137,13 +191,13 @@ Capability flags let the host reason about what your plug-in supports: } ``` -## 10. Packaging & Delivery +## 11. Packaging & Delivery - Output assembly should follow `StellaOps.Authority.Plugin..dll` so the host’s search pattern picks it up. - Place the compiled DLL plus dependencies under `PluginBinaries/Authority` for offline deployments; include hashes/signatures in release notes (Security Guild guidance forthcoming). - Document any external prerequisites (e.g., CA cert bundle) in your plug-in README. - Update `etc/authority.plugins/.yaml` samples and include deterministic SHA256 hashes for optional bootstrap payloads when distributing Offline Kit artefacts. -## 11. Checklist & Handoff +## 12. Checklist & Handoff - ✅ Capabilities declared and validated in automated tests. - ✅ Bootstrap workflows documented (if `bootstrap` capability used) and repeatable. - ✅ Local smoke test + unit/integration suites green (`dotnet test`). @@ -151,7 +205,4 @@ Capability flags let the host reason about what your plug-in supports: - Submit the developer guide update referencing PLG6/DOC4 and tag DevEx + Docs reviewers for sign-off. --- -**Next documentation actions:** -- Add rendered architectural diagram (PlantUML/mermaid) reflecting the lifecycle above once the Docs toolkit pipeline is ready. -- Reference the LDAP RFC (`docs/rfcs/authority-plugin-ldap.md`) in the capability section once review completes. -- Sync terminology with `docs/11_AUTHORITY.md` when that chapter is published to keep glossary terms consistent. +Mermaid sources for the embedded diagrams live under `docs/assets/authority/`. Regenerate the SVG assets with your preferred renderer before committing future updates so the visuals stay in sync with the `.mmd` definitions. diff --git a/docs/dev/fixtures.md b/docs/dev/fixtures.md new file mode 100644 index 00000000..06c29828 --- /dev/null +++ b/docs/dev/fixtures.md @@ -0,0 +1,34 @@ +# Feedser Fixture Maintenance + +Feedser uses a handful of deterministic fixtures to keep connector regressions in check. This guide lists the +fixture sets, where they live, and how to regenerate them safely. + +--- + +## GHSA ↔ OSV parity fixtures + +- **Location:** `src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.*.json` +- **Purpose:** Exercised by `OsvGhsaParityRegressionTests` to ensure OSV + GHSA outputs stay aligned on aliases, + ranges, references, and credits. +- **Regeneration:** Either run the test harness with online regeneration (`UPDATE_PARITY_FIXTURES=1 dotnet test src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj`) + or execute the fixture updater (`dotnet run --project tools/FixtureUpdater/FixtureUpdater.csproj`). Both paths + normalise timestamps and canonical ordering. +- **Verification:** Inspect the diff, then re-run `dotnet test src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj` to confirm parity. + +## GHSA credit parity fixtures + +- **Location:** `src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/credit-parity.{ghsa,osv,nvd}.json` +- **Purpose:** Exercised by `GhsaCreditParityRegressionTests` to guarantee GHSA/NVD/OSV acknowledgements remain in lockstep. +- **Regeneration:** `dotnet run --project tools/FixtureUpdater/FixtureUpdater.csproj` rewrites all three canonical snapshots. +- **Verification:** `dotnet test src/StellaOps.Feedser.Source.Ghsa.Tests/StellaOps.Feedser.Source.Ghsa.Tests.csproj`. + +> Always commit fixture changes together with the code that motivated them and reference the regression test that guards the behaviour. + +## Apple security update fixtures + +- **Location:** `src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/*.html` and `.expected.json`. +- **Purpose:** Exercised by `AppleLiveRegressionTests` to guarantee the Apple HTML parser and mapper stay deterministic while covering Rapid Security Responses and multi-device advisories. +- **Regeneration:** Use the helper scripts (`scripts/update-apple-fixtures.sh` or `scripts/update-apple-fixtures.ps1`). They export `UPDATE_APPLE_FIXTURES=1`, propagate the flag through `WSLENV`, touch `.update-apple-fixtures`, and then run the Apple test project. This keeps WSL/VSCode test invocations in sync while the refresh workflow fetches live Apple support pages, sanitises them, and rewrites both the HTML and expected DTO snapshots with normalised ordering. +- **Verification:** Inspect the generated diffs and re-run `dotnet test src/StellaOps.Feedser.Source.Vndr.Apple.Tests/StellaOps.Feedser.Source.Vndr.Apple.Tests.csproj` without the env var to confirm determinism. + +> **Tip for other connector owners:** mirror the sentinel + `WSLENV` pattern (`touch .update--fixtures`, append the env var via `WSLENV`) when you add fixture refresh scripts so contributors running under WSL inherit the regeneration flag automatically. diff --git a/docs/dev/merge_semver_playbook.md b/docs/dev/merge_semver_playbook.md index 56774b7d..7ae3044a 100644 --- a/docs/dev/merge_semver_playbook.md +++ b/docs/dev/merge_semver_playbook.md @@ -29,6 +29,8 @@ var rule = primitive.ToNormalizedVersionRule(notes: "nvd:CVE-2025-1234"); // rule => scheme=semver, type=range, min=1.2.3, minInclusive=true, max=2.0.0, maxInclusive=false ``` +If you omit the optional `notes` argument, `ToNormalizedVersionRule` now falls back to the primitive’s `ConstraintExpression`, ensuring the original comparator expression is preserved for provenance/audit queries. + Emit the resulting rule inside `AffectedPackage.NormalizedVersions` while continuing to populate `AffectedVersionRange.RangeExpression` for backward compatibility. ## 3. Merge dedupe flow @@ -97,3 +99,56 @@ Follow the operational checklist in `docs/ops/migrations/SEMVER_STYLE.md`. The s - [ ] Confirm integration tests include fixtures with normalized rules and SemVer styles. For deeper query examples and maintenance tasks, continue with [Normalized Versions Query Guide](mongo_indices.md). + +## 8. Storage projection reference + +`NormalizedVersionDocumentFactory` copies each normalized rule into MongoDB using the shape below. Use this as a contract when reviewing connector fixtures or diagnosing merge/storage diffs: + +```json +{ + "packageId": "pkg:npm/example", + "packageType": "npm", + "scheme": "semver", + "type": "range", + "style": "range", + "min": "1.2.3", + "minInclusive": true, + "max": "2.0.0", + "maxInclusive": false, + "value": null, + "notes": "ghsa:GHSA-xxxx-yyyy", + "decisionReason": "ghsa-precedence-over-nvd", + "constraint": ">= 1.2.3 < 2.0.0", + "source": "ghsa", + "recordedAt": "2025-10-11T00:00:00Z" +} +``` + +For distro-specific ranges (`nevra`, `evr`) the same envelope applies with `scheme` switched accordingly. Example: + +```json +{ + "packageId": "bash", + "packageType": "rpm", + "scheme": "nevra", + "type": "range", + "style": "range", + "min": "0:4.4.18-2.el7", + "minInclusive": true, + "max": "0:4.4.20-1.el7", + "maxInclusive": false, + "value": null, + "notes": "redhat:RHSA-2025:1234", + "decisionReason": "rhel-priority-over-nvd", + "constraint": "<= 0:4.4.20-1.el7", + "source": "redhat", + "recordedAt": "2025-10-11T00:00:00Z" +} +``` + +If a new scheme is required (for example, `apple.build` or `ios.semver`), raise it with the Models team before emitting documents so merge comparers and hashing logic can incorporate the change deterministically. + +## 9. Observability signals + +- `feedser.merge.normalized_rules` (counter, tags: `package_type`, `scheme`) – increments once per normalized rule retained after precedence merge. +- `feedser.merge.normalized_rules_missing` (counter, tags: `package_type`) – increments when a merged package still carries version ranges but no normalized rules; watch for spikes to catch connectors that have not emitted normalized arrays yet. diff --git a/docs/dev/mongo_indices.md b/docs/dev/mongo_indices.md index 16a2d9ad..53992628 100644 --- a/docs/dev/mongo_indices.md +++ b/docs/dev/mongo_indices.md @@ -2,6 +2,8 @@ This guide complements the Sprint 1–2 normalized versions rollout. It documents recommended indexes and aggregation patterns for querying `AffectedPackage.normalizedVersions`. +For a field-by-field look at how normalized rules persist in MongoDB (including provenance metadata), see Section 8 of the [Feedser SemVer Merge Playbook](merge_semver_playbook.md). + ## 1. Recommended indexes When `feedser.storage.enableSemVerStyle` is enabled, advisories expose a flattened diff --git a/docs/dev/normalized_versions_rollout.md b/docs/dev/normalized_versions_rollout.md new file mode 100644 index 00000000..6373a162 --- /dev/null +++ b/docs/dev/normalized_versions_rollout.md @@ -0,0 +1,49 @@ +# Normalized Versions Rollout Dashboard (Sprint 2 – Feedser) + +_Status date: 2025-10-12 17:05 UTC_ + +This dashboard tracks connector readiness for emitting `AffectedPackage.NormalizedVersions` arrays and highlights upcoming coordination checkpoints. Use it alongside: + +- [`src/StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md`](../../src/StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md) for detailed guidance and timelines. +- [Feedser SemVer Merge Playbook](merge_semver_playbook.md) §8 for persisted Mongo document shapes. +- [Normalized Versions Query Guide](mongo_indices.md) for index/query validation steps. + +## Key milestones + +- **2025-10-13** – Normalization to finalize `SemVerRangeRuleBuilder` API contract for review. +- **2025-10-17** – Connector owners to post fixture PRs showing `NormalizedVersions` arrays (even if feature-flagged). +- **2025-10-18** – Merge cross-connector review to validate consistent field usage before enabling union logic. + +## Connector readiness matrix + +| Connector | Owner team | Normalized versions status | Last update | Next action / link | +|-----------|------------|---------------------------|-------------|--------------------| +| Acsc | BE-Conn-ACSC | ❌ Not started – mapper pending | 2025-10-11 | Design DTOs + mapper with normalized rule array; see `src/StellaOps.Feedser.Source.Acsc/TASKS.md`. | +| Cccs | BE-Conn-CCCS | ❌ Not started – mapper pending | 2025-10-11 | Add normalized SemVer array in canonical mapper; coordinate fixtures per `TASKS.md`. | +| CertBund | BE-Conn-CERTBUND | ❌ Not started – mapper pending | 2025-10-11 | Capture firmware-style ranges; emit normalized payload; `src/StellaOps.Feedser.Source.CertBund/TASKS.md`. | +| CertCc | BE-Conn-CERTCC | ⚠️ In progress – fetch pipeline DOING | 2025-10-11 | Implement VINCE mapper with SemVer/NEVRA rules; unblock snapshot regeneration; `src/StellaOps.Feedser.Source.CertCc/TASKS.md`. | +| Kev | BE-Conn-KEV | ✅ Normalized catalog/due-date rules verified | 2025-10-12 | Fixtures reconfirmed via `dotnet test src/StellaOps.Feedser.Source.Kev.Tests`; `src/StellaOps.Feedser.Source.Kev/TASKS.md`. | +| Cve | BE-Conn-CVE | ✅ Normalized SemVer rules verified | 2025-10-12 | Snapshot parity green (`dotnet test src/StellaOps.Feedser.Source.Cve.Tests`); `src/StellaOps.Feedser.Source.Cve/TASKS.md`. | +| Ghsa | BE-Conn-GHSA | ⚠️ DOING – normalized rollout task active | 2025-10-11 18:45 UTC | Wire `SemVerRangeRuleBuilder` + refresh fixtures; `src/StellaOps.Feedser.Source.Ghsa/TASKS.md`. | +| Osv | BE-Conn-OSV | ✅ SemVer mapper & parity fixtures verified | 2025-10-12 | GHSA parity regression passing (`dotnet test src/StellaOps.Feedser.Source.Osv.Tests`); `src/StellaOps.Feedser.Source.Osv/TASKS.md`. | +| Ics.Cisa | BE-Conn-ICS-CISA | ❌ Not started – mapper TODO | 2025-10-11 | Plan SemVer/firmware scheme selection; `src/StellaOps.Feedser.Source.Ics.Cisa/TASKS.md`. | +| Kisa | BE-Conn-KISA | ❌ Not started – mapper TODO | 2025-10-11 | Localisation-aware mapper with normalized rules; `src/StellaOps.Feedser.Source.Kisa/TASKS.md`. | +| Ru.Bdu | BE-Conn-BDU | ❌ Not started – mapper TODO | 2025-10-11 | Emit normalized ranges, capture provenance; `src/StellaOps.Feedser.Source.Ru.Bdu/TASKS.md`. | +| Ru.Nkcki | BE-Conn-Nkcki | ❌ Not started – mapper TODO | 2025-10-11 | Similar to BDU; ensure Cyrillic provenance preserved; `src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md`. | +| Vndr.Apple | BE-Conn-Apple | ✅ Shipped – emitting normalized arrays | 2025-10-11 | Continue fixture/tooling work; `src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md`. | +| Vndr.Cisco | BE-Conn-Cisco | ❌ Not started – mapper TODO | 2025-10-11 | Decide on scheme (`semver` vs custom) before emitting rules; `src/StellaOps.Feedser.Source.Vndr.Cisco/TASKS.md`. | +| Vndr.Msrc | BE-Conn-MSRC | ❌ Not started – mapper TODO | 2025-10-11 | Gather samples, define scheme, emit normalized rules; `src/StellaOps.Feedser.Source.Vndr.Msrc/TASKS.md`. | +| Nvd | BE-Conn-NVD | ⚠️ Needs follow-up – mapper complete but normalized array MR pending | 2025-10-11 | Align CVE notes + normalized payload flag; `src/StellaOps.Feedser.Source.Nvd/TASKS.md`. | + +Legend: ✅ complete, ⚠️ in progress/partial, ❌ not started. + +## Monitoring + +- Merge now emits `feedser.merge.normalized_rules` (tags: `package_type`, `scheme`) and `feedser.merge.normalized_rules_missing` (tags: `package_type`). Track these counters to confirm normalized arrays land as connectors roll out. +- Expect `normalized_rules_missing` to trend toward zero as each connector flips on normalized output. Investigate any sustained counts by checking the corresponding module `TASKS.md`. + +## How to use this dashboard + +1. Before opening a connector PR, update the module `TASKS.md` entry and drop a short bullet here (status + timestamp). +2. When a connector lands normalized outputs, flip the status to ✅ and note any rollout toggles (feature flags, fixture regenerations). +3. If a dependency or blocker emerges, add it both in the module `TASKS.md` and in this matrix so merge/storage can escalate quickly. diff --git a/docs/feedser-connector-research-20251011.md b/docs/feedser-connector-research-20251011.md new file mode 100644 index 00000000..37f1db76 --- /dev/null +++ b/docs/feedser-connector-research-20251011.md @@ -0,0 +1,39 @@ +# Feedser Connector Research – 2025-10-11 + +Snapshot of direct network checks performed on 2025-10-11 (UTC) for the national/vendor connectors in scope. Use alongside each module’s `TASKS.md` notes. + +## ACSC (Australia) +- Enumerated feed slugs `/acsc/view-all-content/{alerts,advisories,news,publications,threats}/rss`; every endpoint negotiates HTTP/2 then aborts with `INTERNAL_ERROR` (curl exit 92). Forcing HTTP/1.1 hangs >600 s and sitemap/HTML fetches fail the same way. +- Next actions: prototype `SocketsHttpHandler` settings (`RequestVersionOrLower`, allow fallback to relay), capture successful headers from partner vantage (need retention + cache semantics), and keep `FEEDCONN-SHARED-HTTP2-001` open for downgrade work. + +## CCCS (Canada) +- RSS endpoint (`https://cyber.gc.ca/api/cccs/rss/v1/get?...`) 301s to Atom feed (`/api/cccs/atom/v1/get?...`) with 50-entry window, HTML-heavy `` fields, and no cache headers. +- Next actions: enumerate additional `feed` query values, sanitise inline HTML for DTO storage, and track retention depth via HTML pagination (`?page=`). + +## CERT-Bund (Germany) +- `https://wid.cert-bund.de/content/public/securityAdvisory/rss` responds 200 without cookies (250-item window, German taxonomy). Detail links load an Angular SPA that fetches JSON behind session cookies. +- Next actions: script SPA cookie/bootstrap, discover JSON detail endpoint, and capture advisory schema for parser planning. + +## KISA / KNVD (Korea) +- `https://knvd.krcert.or.kr/rss/securityInfo.do` and `/rss/securityNotice.do` return UTF-8 RSS (10-item window) with `detailDos.do?IDX=` links. No cookies required for feed fetch. +- Next actions: trace SPA detail requests to identify JSON endpoints, normalise Hangul content, and finalise localisation plan. + +## BDU (Russia / FSTEC) +- Candidate endpoints (`https://bdu.fstec.ru/component/rsform/form/7-bdu?format=xml/json`) return 403/404; TLS chain requires Russian Trusted Sub CA and WAF expects additional headers. +- Next actions: acquire official PEM chain, point `feedser:httpClients:source.bdu:trustedRootPaths` (or `feedser:sources:bdu:http:trustedRootPaths`) at the Offline Kit PEM, keep `allowInvalidCertificates=false`, script session bootstrap, then capture RSS/HTML schema for parser work. + +## NKTsKI / cert.gov.ru (Russia) +- `https://cert.gov.ru/rss/advisories.xml` served via Bitrix returns 403/404 even with `Accept-Language: ru-RU`; TLS chain also requires Russian trust anchors. +- Next actions: source trust store, configure `feedser:httpClients:source.nkcki:trustedRootPaths` (Offline Kit root via `feedser:offline:root`), prepare proxy fallback, and once accessible document taxonomy/retention plus attachment handling. + +## CISA ICS (United States) +- `curl -I https://www.cisa.gov/cybersecurity-advisories/ics-advisories.xml` returns HTTP 403 + `x-reference-error` (Akamai). Same for legacy feed paths. +- Next actions: secure GovDelivery access, document token rotation, and build HTML/email fallback with throttling. + +## Cisco PSIRT +- `https://api.cisco.com/security/advisories/latest` returns `ERR_596_SERVICE_NOT_FOUND` when unauthenticated. openVuln REST requires Mashery OAuth (client credentials) with quotas ~5 req/s, 30/min, 5 000/day; supports `pageIndex/pageSize` pagination. +- Next actions: register OAuth app, capture pagination/delta parameters, and compare API vs RSS coverage. + +## Microsoft MSRC +- REST endpoint (`https://api.msrc.microsoft.com/sug/v2.0/en-US/vulnerabilities`) requires Azure AD token + `api-version` (current `2024-08-01`) and supports delta filters (`lastModifiedStartDateTime`). CVRF ZIP remains available for offline use. +- Next actions: finalise AAD app registration, implement token cache, and design combined REST+CVRF ingestion path for determinism. diff --git a/docs/ops/authority-backup-restore.md b/docs/ops/authority-backup-restore.md index f587c467..bef10c8c 100644 --- a/docs/ops/authority-backup-restore.md +++ b/docs/ops/authority-backup-restore.md @@ -80,12 +80,12 @@ docker compose up -d curl -fsS http://localhost:8080/health ``` -6. **Validate JWKS and tokens:** call `/jwks` and issue a short-lived token via the CLI to confirm key material matches expectations. +6. **Validate JWKS and tokens:** call `/jwks` and issue a short-lived token via the CLI to confirm key material matches expectations. If the restored environment requires a fresh signing key, follow the rotation SOP in [`docs/11_AUTHORITY.md`](../11_AUTHORITY.md) using `ops/authority/key-rotation.sh` to invoke `/internal/signing/rotate`. ## Disaster Recovery Notes - **Air-gapped replication:** replicate archives via the Offline Update Kit transport channels; never attach USB devices without scanning. - **Retention:** maintain 30 daily snapshots + 12 monthly archival copies. Rotate encryption keys annually. -- **Key compromise:** if signing keys are suspected compromised, restore from the latest clean backup, rotate via OPS3 (key rotation tooling), and publish a revocation notice. +- **Key compromise:** if signing keys are suspected compromised, restore from the latest clean backup, rotate via OPS3 (see `ops/authority/key-rotation.sh` and `docs/11_AUTHORITY.md`), and publish a revocation notice. - **Mongo version:** keep dump/restore images pinned to the deployment version (compose uses `mongo:7`). Restoring across major versions requires a compatibility review. ## Verification Checklist diff --git a/docs/ops/authority-key-rotation.md b/docs/ops/authority-key-rotation.md new file mode 100644 index 00000000..c2d4b83e --- /dev/null +++ b/docs/ops/authority-key-rotation.md @@ -0,0 +1,83 @@ +# Authority Signing Key Rotation Playbook + +> **Status:** Authored 2025-10-12 as part of OPS3.KEY-ROTATION rollout. +> Use together with `docs/11_AUTHORITY.md` (Authority service guide) and the automation shipped under `ops/authority/`. + +## 1. Overview + +Authority publishes JWKS and revocation bundles signed with ES256 keys. To rotate those keys without downtime we now provide: + +- **Automation script:** `ops/authority/key-rotation.sh` + Shell helper that POSTS to `/internal/signing/rotate`, supports metadata, dry-run, and confirms JWKS afterwards. +- **CI workflow:** `.gitea/workflows/authority-key-rotation.yml` + Manual dispatch workflow that pulls environment-specific secrets, runs the script, and records the result. Works across staging/production by passing the `environment` input. + +This playbook documents the repeatable sequence for all environments. + +## 2. Pre-requisites + +1. **Generate a new PEM key (per environment)** + ```bash + openssl ecparam -name prime256v1 -genkey -noout \ + -out certificates/authority-signing--.pem + chmod 600 certificates/authority-signing--.pem + ``` +2. **Stash the previous key** under the same volume so it can be referenced in `signing.additionalKeys` after rotation. +3. **Ensure secrets/vars exist in Gitea** + - `_AUTHORITY_BOOTSTRAP_KEY` + - `_AUTHORITY_URL` + - Optional shared defaults `AUTHORITY_BOOTSTRAP_KEY`, `AUTHORITY_URL`. + +## 3. Executing the rotation + +### Option A – via CI workflow (recommended) + +1. Navigate to **Actions → Authority Key Rotation**. +2. Provide inputs: + - `environment`: `staging`, `production`, etc. + - `key_id`: new `kid` (e.g. `authority-signing-2025-dev`). + - `key_path`: path as seen by the Authority service (e.g. `../certificates/authority-signing-2025-dev.pem`). + - Optional `metadata`: comma-separated `key=value` pairs (for audit trails). +3. Trigger. The workflow: + - Reads the bootstrap key/URL from secrets. + - Runs `ops/authority/key-rotation.sh`. + - Prints the JWKS response for verification. + +### Option B – manual shell invocation + +```bash +AUTHORITY_BOOTSTRAP_KEY=$(cat /secure/authority-bootstrap.key) \ +./ops/authority/key-rotation.sh \ + --authority-url https://authority.example.com \ + --key-id authority-signing-2025-dev \ + --key-path ../certificates/authority-signing-2025-dev.pem \ + --meta rotatedBy=ops --meta changeTicket=OPS-1234 +``` + +Use `--dry-run` to inspect the payload before execution. + +## 4. Post-rotation checklist + +1. Update `authority.yaml` (or environment-specific overrides): + - Set `signing.activeKeyId` to the new key. + - Set `signing.keyPath` to the new PEM. + - Append the previous key into `signing.additionalKeys`. + - Ensure `keySource`/`provider` match the values passed to the script. +2. Run `stellaops-cli auth revoke export` so revocation bundles are re-signed with the new key. +3. Confirm `/jwks` lists the new `kid` with `status: "active"` and the previous one as `retired`. +4. Archive the old key securely; keep it available until all tokens/bundles signed with it have expired. + +## 5. Development key state + +For the sample configuration (`etc/authority.yaml.sample`) we minted a placeholder dev key: + +- Active: `authority-signing-2025-dev` (`certificates/authority-signing-2025-dev.pem`) +- Retired: `authority-signing-dev` + +Treat these as examples; real environments must maintain their own PEM material. + +## 6. References + +- `docs/11_AUTHORITY.md` – Architecture and rotation SOP (Section 5). +- `docs/ops/authority-backup-restore.md` – Recovery flow referencing this playbook. +- `ops/authority/README.md` – CLI usage and examples. diff --git a/docs/ops/feedser-apple-operations.md b/docs/ops/feedser-apple-operations.md new file mode 100644 index 00000000..d87fa39b --- /dev/null +++ b/docs/ops/feedser-apple-operations.md @@ -0,0 +1,77 @@ +# Feedser Apple Security Update Connector Operations + +This runbook covers staging and production rollout for the Apple security updates connector (`source:vndr-apple:*`), including observability checks and fixture maintenance. + +## 1. Prerequisites + +- Network egress (or mirrored cache) for `https://gdmf.apple.com/v2/pmv` and the Apple Support domain (`https://support.apple.com/`). +- Optional: corporate proxy exclusions for the Apple hosts if outbound traffic is normally filtered. +- Updated configuration (environment variables or `feedser.yaml`) with an `apple` section. Example baseline: + +```yaml +feedser: + sources: + apple: + softwareLookupUri: "https://gdmf.apple.com/v2/pmv" + advisoryBaseUri: "https://support.apple.com/" + localeSegment: "en-us" + maxAdvisoriesPerFetch: 25 + initialBackfill: "120.00:00:00" + modifiedTolerance: "02:00:00" + failureBackoff: "00:05:00" +``` + +> ℹ️ `softwareLookupUri` and `advisoryBaseUri` must stay absolute and aligned with the HTTP allow-list; Feedser automatically adds both hosts to the connector HttpClient. + +## 2. Staging Smoke Test + +1. Deploy the configuration and restart the Feedser workers to ensure the Apple connector options are bound. +2. Trigger a full connector cycle: + - CLI: `stella db jobs run source:vndr-apple:fetch --and-then source:vndr-apple:parse --and-then source:vndr-apple:map` + - REST: `POST /jobs/run { "kind": "source:vndr-apple:fetch", "chain": ["source:vndr-apple:parse", "source:vndr-apple:map"] }` +3. Validate metrics exported under meter `StellaOps.Feedser.Source.Vndr.Apple`: + - `apple.fetch.items` (documents fetched) + - `apple.fetch.failures` + - `apple.fetch.unchanged` + - `apple.parse.failures` + - `apple.map.affected.count` (histogram of affected package counts) +4. Cross-check the shared HTTP counters: + - `feedser.source.http.requests_total{feedser_source="vndr-apple"}` should increase for both index and detail phases. + - `feedser.source.http.failures_total{feedser_source="vndr-apple"}` should remain flat (0) during a healthy run. +5. Inspect the info logs: + - `Apple software index fetch … processed=X newDocuments=Y` + - `Apple advisory parse complete … aliases=… affected=…` + - `Mapped Apple advisory … pendingMappings=0` +6. Confirm MongoDB state: + - `raw_documents` store contains the HT article HTML with metadata (`apple.articleId`, `apple.postingDate`). + - `dtos` store has `schemaVersion="apple.security.update.v1"`. + - `advisories` collection includes keys `HTxxxxxx` with normalized SemVer rules. + - `source_states` entry for `apple` shows a recent `cursor.lastPosted`. + +## 3. Production Monitoring + +- **Dashboards** – Add the following expressions to your Feedser Grafana board (OTLP/Prometheus naming assumed): + - `rate(apple_fetch_items_total[15m])` vs `rate(feedser_source_http_requests_total{feedser_source="vndr-apple"}[15m])` + - `rate(apple_fetch_failures_total[5m])` for error spikes (`severity=warning` at `>0`) + - `histogram_quantile(0.95, rate(apple_map_affected_count_bucket[1h]))` to watch affected-package fan-out + - `increase(apple_parse_failures_total[6h])` to catch parser drift (alerts at `>0`) +- **Alerts** – Page if `rate(apple_fetch_items_total[2h]) == 0` during business hours while other connectors are active. This often indicates lookup feed failures or misconfigured allow-lists. +- **Logs** – Surface warnings `Apple document {DocumentId} missing GridFS payload` or `Apple parse failed`—repeated hits imply storage issues or HTML regressions. +- **Telemetry pipeline** – `StellaOps.Feedser.WebService` now exports `StellaOps.Feedser.Source.Vndr.Apple` alongside existing Feedser meters; ensure your OTEL collector or Prometheus scraper includes it. + +## 4. Fixture Maintenance + +Regression fixtures live under `src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures`. Refresh them whenever Apple reshapes the HT layout or when new platforms appear. + +1. Run the helper script matching your platform: + - Bash: `./scripts/update-apple-fixtures.sh` + - PowerShell: `./scripts/update-apple-fixtures.ps1` +2. Each script exports `UPDATE_APPLE_FIXTURES=1`, updates the `WSLENV` passthrough, and touches `.update-apple-fixtures` so WSL+VS Code test runs observe the flag. The subsequent test execution fetches the live HT articles listed in `AppleFixtureManager`, sanitises the HTML, and rewrites the `.expected.json` DTO snapshots. +3. Review the diff for localisation or nav noise. Once satisfied, re-run the tests without the env var (`dotnet test src/StellaOps.Feedser.Source.Vndr.Apple.Tests/StellaOps.Feedser.Source.Vndr.Apple.Tests.csproj`) to verify determinism. +4. Commit fixture updates together with any parser/mapping changes that motivated them. + +## 5. Known Issues & Follow-up Tasks + +- Apple occasionally throttles anonymous requests after bursts. The connector backs off automatically, but persistent `apple.fetch.failures` spikes might require mirroring the HT content or scheduling wider fetch windows. +- Rapid Security Responses may appear before the general patch notes surface in the lookup JSON. When that happens, the fetch run will log `detailFailures>0`. Collect sample HTML and refresh fixtures to confirm parser coverage. +- Multi-locale content is still under regression sweep (`src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md`). Capture non-`en-us` snapshots once the fixture tooling stabilises. diff --git a/docs/ops/feedser-authority-audit-runbook.md b/docs/ops/feedser-authority-audit-runbook.md new file mode 100644 index 00000000..3262037a --- /dev/null +++ b/docs/ops/feedser-authority-audit-runbook.md @@ -0,0 +1,150 @@ +# Feedser Authority Audit Runbook + +_Last updated: 2025-10-12_ + +This runbook helps operators verify and monitor the StellaOps Feedser ⇆ Authority integration. It focuses on the `/jobs*` surface, which now requires StellaOps Authority tokens, and the corresponding audit/metric signals that expose authentication and bypass activity. + +## 1. Prerequisites + +- Authority integration is enabled in `feedser.yaml` (or via `FEEDSER_AUTHORITY__*` environment variables) with a valid `clientId`, secret, audience, and required scopes. +- OTLP metrics/log exporters are configured (`feedser.telemetry.*`) or container stdout is shipped to your SIEM. +- Operators have access to the Feedser job trigger endpoints via CLI or REST for smoke tests. + +### Configuration snippet + +```yaml +feedser: + authority: + enabled: true + allowAnonymousFallback: false # keep true only during initial rollout + issuer: "https://authority.internal" + audiences: + - "api://feedser" + requiredScopes: + - "feedser.jobs.trigger" + bypassNetworks: + - "127.0.0.1/32" + - "::1/128" + clientId: "feedser-jobs" + clientSecretFile: "/run/secrets/feedser_authority_client" + tokenClockSkewSeconds: 60 + resilience: + enableRetries: true + retryDelays: + - "00:00:01" + - "00:00:02" + - "00:00:05" + allowOfflineCacheFallback: true + offlineCacheTolerance: "00:10:00" +``` + +> Store secrets outside source control. Feedser reads `clientSecretFile` on startup; rotate by updating the mounted file and restarting the service. + +### Resilience tuning + +- **Connected sites:** keep the default 1 s / 2 s / 5 s retry ladder so Feedser retries transient Authority hiccups but still surfaces outages quickly. Leave `allowOfflineCacheFallback=true` so cached discovery/JWKS data can bridge short Pathfinder restarts. +- **Air-gapped/Offline Kit installs:** extend `offlineCacheTolerance` (15–30 minutes) to keep the cached metadata valid between manual synchronisations. You can also disable retries (`enableRetries=false`) if infrastructure teams prefer to handle exponential backoff at the network layer; Feedser will fail fast but keep deterministic logs. +- Feedser resolves these knobs through `IOptionsMonitor`. Edits to `feedser.yaml` are applied on configuration reload; restart the container if you change environment variables or do not have file-watch reloads enabled. + +## 2. Key Signals + +### 2.1 Audit log channel + +Feedser emits structured audit entries via the `Feedser.Authorization.Audit` logger for every `/jobs*` request once Authority enforcement is active. + +``` +Feedser authorization audit route=/jobs/definitions status=200 subject=ops@example.com clientId=feedser-cli scopes=feedser.jobs.trigger bypass=False remote=10.1.4.7 +``` + +| Field | Sample value | Meaning | +|--------------|-------------------------|------------------------------------------------------------------------------------------| +| `route` | `/jobs/definitions` | Endpoint that processed the request. | +| `status` | `200` / `401` / `409` | Final HTTP status code returned to the caller. | +| `subject` | `ops@example.com` | User or service principal subject (falls back to `(anonymous)` when unauthenticated). | +| `clientId` | `feedser-cli` | OAuth client ID provided by Authority ( `(none)` if the token lacked the claim). | +| `scopes` | `feedser.jobs.trigger` | Normalised scope list extracted from token claims; `(none)` if the token carried none. | +| `bypass` | `True` / `False` | Indicates whether the request succeeded because its source IP matched a bypass CIDR. | +| `remote` | `10.1.4.7` | Remote IP recorded from the connection / forwarded header test hooks. | + +Use your logging backend (e.g., Loki) to index the logger name and filter for suspicious combinations: + +- `status=401 AND bypass=True` – bypass network accepted an unauthenticated call (should be temporary during rollout). +- `status=202 AND scopes="(none)"` – a token without scopes triggered a job; tighten client configuration. +- Spike in `clientId="(none)"` – indicates upstream Authority is not issuing `client_id` claims or the CLI is outdated. + +### 2.2 Metrics + +Feedser publishes counters under the OTEL meter `StellaOps.Feedser.WebService.Jobs`. Tags: `job.kind`, `job.trigger`, `job.outcome`. + +| Metric name | Description | PromQL example | +|-------------------------------|----------------------------------------------------|----------------| +| `web.jobs.triggered` | Accepted job trigger requests. | `sum by (job_kind) (rate(web_jobs_triggered_total[5m]))` | +| `web.jobs.trigger.conflict` | Rejected triggers (already running, disabled…). | `sum(rate(web_jobs_trigger_conflict_total[5m]))` | +| `web.jobs.trigger.failed` | Server-side job failures. | `sum(rate(web_jobs_trigger_failed_total[5m]))` | + +> Prometheus/OTEL collectors typically surface counters with `_total` suffix. Adjust queries to match your pipeline’s generated metric names. + +Correlate audit logs with the following global meter exported via `Feedser.SourceDiagnostics`: + +- `feedser.source.http.requests_total{feedser_source="jobs-run"}` – ensures REST/manual triggers route through Authority. +- If Grafana dashboards are deployed, extend the “Feedser Jobs” board with the above counters plus a table of recent audit log entries. + +## 3. Alerting Guidance + +1. **Unauthorized bypass attempt** + - Query: `sum(rate(log_messages_total{logger="Feedser.Authorization.Audit", status="401", bypass="True"}[5m])) > 0` + - Action: verify `bypassNetworks` list; confirm expected maintenance windows; rotate credentials if suspicious. + +2. **Missing scopes** + - Query: `sum(rate(log_messages_total{logger="Feedser.Authorization.Audit", scopes="(none)", status="200"}[5m])) > 0` + - Action: audit Authority client registration; ensure `requiredScopes` includes `feedser.jobs.trigger`. + +3. **Trigger failure surge** + - Query: `sum(rate(web_jobs_trigger_failed_total[10m])) > 0` with severity `warning` if sustained for 10 minutes. + - Action: inspect correlated audit entries and `Feedser.Telemetry` traces for job execution errors. + +4. **Conflict spike** + - Query: `sum(rate(web_jobs_trigger_conflict_total[10m])) > 5` (tune threshold). + - Action: downstream scheduling may be firing repetitive triggers; ensure precedence is configured properly. + +5. **Authority offline** + - Watch `Feedser.Authorization.Audit` logs for `status=503` or `status=500` along with `clientId="(none)"`. Investigate Authority availability before re-enabling anonymous fallback. + +## 4. Rollout & Verification Procedure + +1. **Pre-checks** + - Confirm `allowAnonymousFallback` is `false` in production; keep `true` only during staged validation. + - Validate Authority issuer metadata is reachable from Feedser (`curl https://authority.internal/.well-known/openid-configuration` from the host). + +2. **Smoke test with valid token** + - Obtain a token via CLI: `stella auth login --scope feedser.jobs.trigger`. + - Trigger a read-only endpoint: `curl -H "Authorization: Bearer $TOKEN" https://feedser.internal/jobs/definitions`. + - Expect HTTP 200/202 and an audit log with `bypass=False`, `scopes=feedser.jobs.trigger`. + +3. **Negative test without token** + - Call the same endpoint without a token. Expect HTTP 401, `bypass=False`. + - If the request succeeds, double-check `bypassNetworks` and ensure fallback is disabled. + +4. **Bypass check (if applicable)** + - From an allowed maintenance IP, call `/jobs/definitions` without a token. Confirm the audit log shows `bypass=True`. Review business justification and expiry date for such entries. + +5. **Metrics validation** + - Ensure `web.jobs.triggered` counter increments during accepted runs. + - Exporters should show corresponding spans (`feedser.job.trigger`) if tracing is enabled. + +## 5. Troubleshooting + +| Symptom | Probable cause | Remediation | +|---------|----------------|-------------| +| Audit log shows `clientId=(none)` for all requests | Authority not issuing `client_id` claim or CLI outdated | Update StellaOps Authority configuration (`StellaOpsAuthorityOptions.Token.Claims.ClientId`), or upgrade the CLI token acquisition flow. | +| Requests succeed with `bypass=True` unexpectedly | Local network added to `bypassNetworks` or fallback still enabled | Remove/adjust the CIDR list, disable anonymous fallback, restart Feedser. | +| HTTP 401 with valid token | `requiredScopes` missing from client registration or token audience mismatch | Verify Authority client scopes (`feedser.jobs.trigger`) and ensure the token audience matches `audiences` config. | +| Metrics missing from Prometheus | Telemetry exporters disabled or filter missing OTEL meter | Set `feedser.telemetry.enableMetrics=true`, ensure collector includes `StellaOps.Feedser.WebService.Jobs` meter. | +| Sudden spike in `web.jobs.trigger.failed` | Downstream job failure or Authority timeout mid-request | Inspect Feedser job logs, re-run with tracing enabled, validate Authority latency. | + +## 6. References + +- `docs/21_INSTALL_GUIDE.md` – Authority configuration quick start. +- `docs/17_SECURITY_HARDENING_GUIDE.md` – Security guardrails and enforcement deadlines. +- `docs/ops/authority-monitoring.md` – Authority-side monitoring and alerting playbook. +- `StellaOps.Feedser.WebService/Filters/JobAuthorizationAuditFilter.cs` – source of audit log fields. diff --git a/docs/ops/feedser-conflict-resolution.md b/docs/ops/feedser-conflict-resolution.md index 255f09a7..5a38d106 100644 --- a/docs/ops/feedser-conflict-resolution.md +++ b/docs/ops/feedser-conflict-resolution.md @@ -47,6 +47,12 @@ Expect all logs at `Information`. Ensure OTEL exporters include the scope `Stell 3. **Job health** - `stellaops-cli db merge` exit code `1` signifies unresolved conflicts. Pipe to automation that captures logs and notifies #feedser-ops. +### Threshold updates (2025-10-12) + +- `feedser.merge.conflicts` – Page only when ≥ 2 events fire within 30 minutes; the synthetic conflict fixture run produces 0 conflicts, so the first event now routes to Slack for manual review instead of paging. +- `feedser.merge.overrides` – Raise a warning when the 30-minute sum exceeds 10 (canonical triple yields exactly 1 summary override with `primary_source=osv`, `suppressed_source=ghsa`). +- `feedser.merge.range_overrides` – Maintain the 15-minute alert at ≥ 3 but annotate dashboards that the regression triple emits a single `package_type=semver` override so ops can spot unexpected spikes. + --- ## 4. Triage Workflow @@ -128,3 +134,19 @@ Expect all logs at `Information`. Ensure OTEL exporters include the scope `Stell - Storage audit trail: `src/StellaOps.Feedser.Merge/Services/MergeEventWriter.cs`, `src/StellaOps.Feedser.Storage.Mongo/MergeEvents`. Keep this runbook synchronized with future sprint notes and update alert thresholds as baseline volumes change. + +--- + +## 9. Synthetic Regression Fixtures + +- **Locations** – Canonical conflict snapshots now live at `src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/conflict-ghsa.canonical.json`, `src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/conflict-nvd.canonical.json`, and `src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/conflict-osv.canonical.json`. +- **Validation commands** – To regenerate and verify the fixtures offline, run: + +```bash +dotnet test src/StellaOps.Feedser.Source.Ghsa.Tests/StellaOps.Feedser.Source.Ghsa.Tests.csproj --filter GhsaConflictFixtureTests +dotnet test src/StellaOps.Feedser.Source.Nvd.Tests/StellaOps.Feedser.Source.Nvd.Tests.csproj --filter NvdConflictFixtureTests +dotnet test src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj --filter OsvConflictFixtureTests +dotnet test src/StellaOps.Feedser.Merge.Tests/StellaOps.Feedser.Merge.Tests.csproj --filter MergeAsync_AppliesCanonicalRulesAndPersistsDecisions +``` + +- **Expected signals** – The triple produces one freshness-driven summary override (`primary_source=osv`, `suppressed_source=ghsa`) and one range override for the npm SemVer package while leaving `feedser.merge.conflicts` at zero. Use these values as the baseline when tuning dashboards or load-testing alert pipelines. diff --git a/docs/ops/feedser-cve-kev-grafana-dashboard.json b/docs/ops/feedser-cve-kev-grafana-dashboard.json new file mode 100644 index 00000000..51df8cc4 --- /dev/null +++ b/docs/ops/feedser-cve-kev-grafana-dashboard.json @@ -0,0 +1,151 @@ +{ + "title": "Feedser CVE & KEV Observability", + "uid": "feedser-cve-kev", + "schemaVersion": 38, + "version": 1, + "editable": true, + "timezone": "", + "time": { + "from": "now-24h", + "to": "now" + }, + "refresh": "5m", + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "query": "prometheus", + "refresh": 1, + "hide": 0 + } + ] + }, + "panels": [ + { + "type": "timeseries", + "title": "CVE fetch success vs failure", + "gridPos": { "h": 9, "w": 12, "x": 0, "y": 0 }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "rate(cve_fetch_success_total[5m])", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "legendFormat": "success" + }, + { + "refId": "B", + "expr": "rate(cve_fetch_failures_total[5m])", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "legendFormat": "failure" + } + ] + }, + { + "type": "timeseries", + "title": "KEV fetch cadence", + "gridPos": { "h": 9, "w": 12, "x": 12, "y": 0 }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "rate(kev_fetch_success_total[30m])", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "legendFormat": "success" + }, + { + "refId": "B", + "expr": "rate(kev_fetch_failures_total[30m])", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "legendFormat": "failure" + }, + { + "refId": "C", + "expr": "rate(kev_fetch_unchanged_total[30m])", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "legendFormat": "unchanged" + } + ] + }, + { + "type": "table", + "title": "KEV parse anomalies (24h)", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (reason) (increase(kev_parse_anomalies_total[24h]))", + "format": "table", + "datasource": { "type": "prometheus", "uid": "${datasource}" } + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "renameByName": { + "Value": "count" + } + } + } + ] + }, + { + "type": "timeseries", + "title": "Advisories emitted", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "rate(cve_map_success_total[15m])", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "legendFormat": "CVE" + }, + { + "refId": "B", + "expr": "rate(kev_map_advisories_total[24h])", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "legendFormat": "KEV" + } + ] + } + ] +} diff --git a/docs/ops/feedser-cve-kev-operations.md b/docs/ops/feedser-cve-kev-operations.md index 8635933c..3d042ec1 100644 --- a/docs/ops/feedser-cve-kev-operations.md +++ b/docs/ops/feedser-cve-kev-operations.md @@ -34,19 +34,22 @@ feedser: - Feedser CLI: `stella db jobs run source:cve:fetch --and-then source:cve:parse --and-then source:cve:map` - REST fallback: `POST /jobs/run { "kind": "source:cve:fetch", "chain": ["source:cve:parse", "source:cve:map"] }` 3. Observe the following metrics (exported via OTEL meter `StellaOps.Feedser.Source.Cve`): - - `cve.fetch.attempts`, `cve.fetch.success`, `cve.fetch.failures`, `cve.fetch.unchanged` + - `cve.fetch.attempts`, `cve.fetch.success`, `cve.fetch.documents`, `cve.fetch.failures`, `cve.fetch.unchanged` - `cve.parse.success`, `cve.parse.failures`, `cve.parse.quarantine` - `cve.map.success` -4. Verify the MongoDB advisory store contains fresh CVE advisories (`advisoryKey` prefix `cve/`) and that the source cursor (`source_states` collection) advanced. +4. Verify Prometheus shows matching `feedser.source.http.requests_total{feedser_source="cve"}` deltas (list vs detail phases) while `feedser.source.http.failures_total{feedser_source="cve"}` stays flat. +5. Confirm the info-level summary log `CVEs fetch window … pages=X detailDocuments=Y detailFailures=Z` appears once per fetch run and shows `detailFailures=0`. +6. Verify the MongoDB advisory store contains fresh CVE advisories (`advisoryKey` prefix `cve/`) and that the source cursor (`source_states` collection) advanced. ### 1.3 Production Monitoring -- **Dashboards** – Add the counters above plus `feedser.range.primitives` (filtered by `scheme=semver` or `scheme=vendor`) to the Feedser overview board. Alert when: - - `rate(cve.fetch.failures[5m]) > 0` - - `rate(cve.map.success[15m]) == 0` while fetch attempts continue - - `sum_over_time(cve.parse.quarantine[1h]) > 0` -- **Logs** – Watch for `CveConnector` warnings such as `Failed fetching CVE record` or schema validation errors (`Malformed CVE JSON`). These are emitted with the CVE ID and document identifier for triage. -- **Backfill window** – operators can tighten or widen the `initialBackfill` / `maxPagesPerFetch` values after validating baseline throughput. Update the config and restart the worker to apply changes. +- **Dashboards** – Plot `rate(cve_fetch_success_total[5m])`, `rate(cve_fetch_failures_total[5m])`, and `rate(cve_fetch_documents_total[5m])` alongside `feedser_source_http_requests_total{feedser_source="cve"}` to confirm HTTP and connector counters stay aligned. Keep `feedser.range.primitives{scheme=~"semver|vendor"}` on the same board for range coverage. Example alerts: + - `rate(cve_fetch_failures_total[5m]) > 0` for 10 minutes (`severity=warning`) + - `rate(cve_map_success_total[15m]) == 0` while `rate(cve_fetch_success_total[15m]) > 0` (`severity=critical`) + - `sum_over_time(cve_parse_quarantine_total[1h]) > 0` to catch schema anomalies +- **Logs** – Monitor warnings such as `Failed fetching CVE record {CveId}` and `Malformed CVE JSON`, and surface the summary info log `CVEs fetch window … detailFailures=0 detailUnchanged=0` on dashboards. A non-zero `detailFailures` usually indicates rate-limit or auth issues on detail requests. +- **Grafana pack** – Import `docs/ops/feedser-cve-kev-grafana-dashboard.json` and filter by panel legend (`CVE`, `KEV`) to reuse the canned layout. +- **Backfill window** – Operators can tighten or widen `initialBackfill` / `maxPagesPerFetch` after validating throughput. Update config and restart Feedser to apply changes. ## 2. CISA KEV Connector (`source:kev:*`) @@ -67,7 +70,15 @@ feedser: ### 2.2 Schema validation & anomaly handling -From this sprint the connector validates the KEV JSON payload against `Schemas/kev-catalog.schema.json`. Malformed documents are quarantined, and entries missing a CVE ID are dropped with a warning (`reason=missingCveId`). Operators should treat repeated schema failures as an upstream regression and coordinate with CISA or mirror maintainers. +The connector validates each catalog against `Schemas/kev-catalog.schema.json`. Failures increment `kev.parse.failures_total{reason="schema"}` and the document is quarantined (status `Failed`). Additional failure reasons include `download`, `invalidJson`, `deserialize`, `missingPayload`, and `emptyCatalog`. Entry-level anomalies are surfaced through `kev.parse.anomalies_total` with reasons: + +| Reason | Meaning | +| --- | --- | +| `missingCveId` | Catalog entry omitted `cveID`; the entry is skipped. | +| `countMismatch` | Catalog `count` field disagreed with the actual entry total. | +| `nullEntry` | Upstream emitted a `null` entry object (rare upstream defect). | + +Treat repeated schema failures or growing anomaly counts as an upstream regression and coordinate with CISA or mirror maintainers. ### 2.3 Smoke Test (staging) @@ -79,13 +90,16 @@ From this sprint the connector validates the KEV JSON payload against `Schemas/k - `kev.fetch.attempts`, `kev.fetch.success`, `kev.fetch.unchanged`, `kev.fetch.failures` - `kev.parse.entries` (tag `catalogVersion`), `kev.parse.failures`, `kev.parse.anomalies` (tag `reason`) - `kev.map.advisories` (tag `catalogVersion`) -4. Confirm MongoDB documents exist for the catalog JSON (`raw_documents` & `dtos`) and that advisories with prefix `kev/` are written. +4. Confirm `feedser.source.http.requests_total{feedser_source="kev"}` increments once per fetch and that the paired `feedser.source.http.failures_total` stays flat (zero increase). +5. Inspect the info logs `Fetched KEV catalog document … pendingDocuments=…` and `Parsed KEV catalog document … entries=…`—they should appear exactly once per run and `Mapped X/Y… skipped=0` should match the `kev.map.advisories` delta. +6. Confirm MongoDB documents exist for the catalog JSON (`raw_documents` & `dtos`) and that advisories with prefix `kev/` are written. ### 2.4 Production Monitoring -- Alert when `kev.fetch.success` goes to zero for longer than the expected daily cadence (default: trigger if `rate(kev.fetch.success[8h]) == 0` during business hours). -- Track anomaly spikes via `kev.parse.anomalies{reason="missingCveId"}`. A sustained non-zero rate means the upstream catalog contains unexpected records. -- The connector logs each validated catalog: `Parsed KEV catalog document … entries=X`. Absence of that log alongside consecutive `kev.fetch.success` counts suggests schema validation failures—correlate with warning-level events in the `StellaOps.Feedser.Source.Kev` logger. +- Alert when `rate(kev_fetch_success_total[8h]) == 0` during working hours (daily cadence breach) and when `increase(kev_fetch_failures_total[1h]) > 0`. +- Page the on-call if `increase(kev_parse_failures_total{reason="schema"}[6h]) > 0`—this usually signals an upstream payload change. Treat repeated `reason="download"` spikes as networking issues to the mirror. +- Track anomaly spikes through `sum_over_time(kev_parse_anomalies_total{reason="missingCveId"}[24h])`. Rising `countMismatch` trends point to catalog publishing bugs. +- Surface the fetch/mapping info logs (`Fetched KEV catalog document …` and `Mapped X/Y KEV advisories … skipped=S`) on dashboards; absence of those logs while metrics show success typically means schema validation short-circuited the run. ### 2.5 Known good dashboard tiles @@ -93,12 +107,14 @@ Add the following panels to the Feedser observability board: | Metric | Recommended visualisation | |--------|---------------------------| -| `kev.fetch.success` | Single-stat (last 24 h) with threshold alert | -| `rate(kev.parse.entries[1h])` by `catalogVersion` | Stacked area – highlights daily release size | -| `sum_over_time(kev.parse.anomalies[1d])` by `reason` | Table – anomaly breakdown | +| `rate(kev_fetch_success_total[30m])` | Single-stat (last 24 h) with warning threshold `>0` | +| `rate(kev_parse_entries_total[1h])` by `catalogVersion` | Stacked area – highlights daily release size | +| `sum_over_time(kev_parse_anomalies_total[1d])` by `reason` | Table – anomaly breakdown (matches dashboard panel) | +| `rate(cve_map_success_total[15m])` vs `rate(kev_map_advisories_total[24h])` | Comparative timeseries for advisories emitted | ## 3. Runbook updates - Record staging/production smoke test results (date, catalog version, advisory counts) in your team’s change log. - Add the CVE/KEV job kinds to the standard maintenance checklist so operators can manually trigger them after planned downtime. - Keep this document in sync with future connector changes (for example, new anomaly reasons or additional metrics). +- Version-control dashboard tweaks alongside `docs/ops/feedser-cve-kev-grafana-dashboard.json` so operations can re-import the observability pack during restores. diff --git a/docs/ops/feedser-ghsa-operations.md b/docs/ops/feedser-ghsa-operations.md new file mode 100644 index 00000000..a1767ba9 --- /dev/null +++ b/docs/ops/feedser-ghsa-operations.md @@ -0,0 +1,111 @@ +# Feedser GHSA Connector – Operations Runbook + +_Last updated: 2025-10-12_ + +## 1. Overview +The GitHub Security Advisories (GHSA) connector pulls advisory metadata from the GitHub REST API `/security/advisories` endpoint. GitHub enforces both primary and secondary rate limits, so operators must monitor usage and configure retries to avoid throttling incidents. + +## 2. Rate-limit telemetry +The connector now surfaces rate-limit headers on every fetch and exposes the following metrics via OpenTelemetry: + +| Metric | Description | Tags | +|--------|-------------|------| +| `ghsa.ratelimit.limit` (histogram) | Samples the reported request quota at fetch time. | `phase` = `list` or `detail`, `resource` (e.g., `core`). | +| `ghsa.ratelimit.remaining` (histogram) | Remaining requests returned by `X-RateLimit-Remaining`. | `phase`, `resource`. | +| `ghsa.ratelimit.reset_seconds` (histogram) | Seconds until `X-RateLimit-Reset`. | `phase`, `resource`. | +| `ghsa.ratelimit.exhausted` (counter) | Incremented whenever GitHub returns a zero remaining quota and the connector delays before retrying. | `phase`. | + +### Dashboards & alerts +- Plot `ghsa.ratelimit.remaining` as the latest value to watch the runway. Alert when the value stays below **`RateLimitWarningThreshold`** (default `500`) for more than 5 minutes. +- Raise a separate alert on `increase(ghsa.ratelimit.exhausted[15m]) > 0` to catch hard throttles. +- Overlay `ghsa.fetch.attempts` vs `ghsa.fetch.failures` to confirm retries are effective. + +## 3. Logging signals +When `X-RateLimit-Remaining` falls below `RateLimitWarningThreshold`, the connector emits: +``` +GHSA rate limit warning: remaining {Remaining}/{Limit} for {Phase} {Resource} +``` +When GitHub reports zero remaining calls, the connector logs and sleeps for the reported `Retry-After`/`X-RateLimit-Reset` interval (falling back to `SecondaryRateLimitBackoff`). + +## 4. Configuration knobs (`feedser.yaml`) +```yaml +feedser: + sources: + ghsa: + apiToken: "${GITHUB_PAT}" + pageSize: 50 + requestDelay: "00:00:00.200" + failureBackoff: "00:05:00" + rateLimitWarningThreshold: 500 # warn below this many remaining calls + secondaryRateLimitBackoff: "00:02:00" # fallback delay when GitHub omits Retry-After +``` + +### Recommendations +- Increase `requestDelay` in air-gapped or burst-heavy deployments to smooth token consumption. +- Lower `rateLimitWarningThreshold` only if your dashboards already page on the new histogram; never set it negative. +- For bots using a low-privilege PAT, keep `secondaryRateLimitBackoff` at ≥60 seconds to respect GitHub’s secondary-limit guidance. + +#### Default job schedule + +| Job kind | Cron | Timeout | Lease | +|----------|------|---------|-------| +| `source:ghsa:fetch` | `1,11,21,31,41,51 * * * *` | 6 minutes | 4 minutes | +| `source:ghsa:parse` | `3,13,23,33,43,53 * * * *` | 5 minutes | 4 minutes | +| `source:ghsa:map` | `5,15,25,35,45,55 * * * *` | 5 minutes | 4 minutes | + +These defaults spread GHSA stages across the hour so fetch completes before parse/map fire. Override them via `feedser.jobs.definitions[...]` when coordinating multiple connectors on the same runner. + +## 5. Provisioning credentials + +Feedser requires a GitHub personal access token (classic) with the **`read:org`** and **`security_events`** scopes to pull GHSA data. Store it as a secret and reference it via `feedser.sources.ghsa.apiToken`. + +### Docker Compose (stack operators) +```yaml +services: + feedser: + environment: + FEEDSER__SOURCES__GHSA__APITOKEN: /run/secrets/ghsa_pat + secrets: + - ghsa_pat + +secrets: + ghsa_pat: + file: ./secrets/ghsa_pat.txt # contains only the PAT value +``` + +### Helm values (cluster operators) +```yaml +feedser: + extraEnv: + - name: FEEDSER__SOURCES__GHSA__APITOKEN + valueFrom: + secretKeyRef: + name: feedser-ghsa + key: apiToken + +extraSecrets: + feedser-ghsa: + apiToken: "" +``` + +After rotating the PAT, restart the Feedser workers (or run `kubectl rollout restart deployment/feedser`) to ensure the configuration reloads. + +When enabling GHSA the first time, run a staged backfill: + +1. Trigger `source:ghsa:fetch` manually (CLI or API) outside of peak hours. +2. Watch `feedser.jobs.health` for the GHSA jobs until they report `healthy`. +3. Allow the scheduled cron cadence to resume once the initial backlog drains (typically < 30 minutes). + +## 6. Runbook steps when throttled +1. Check `ghsa.ratelimit.exhausted` for the affected phase (`list` vs `detail`). +2. Confirm the connector is delaying—logs will show `GHSA rate limit exhausted...` with the chosen backoff. +3. If rate limits stay exhausted: + - Verify no other jobs are sharing the PAT. + - Temporarily reduce `MaxPagesPerFetch` or `PageSize` to shrink burst size. + - Consider provisioning a dedicated PAT (GHSA permissions only) for Feedser. +4. After the quota resets, reset `rateLimitWarningThreshold`/`requestDelay` to their normal values and monitor the histograms for at least one hour. + +## 7. Alert integration quick reference +- Prometheus: `ghsa_ratelimit_remaining_bucket` (from histogram) – use `histogram_quantile(0.99, ...)` to trend capacity. +- VictoriaMetrics: `LAST_over_time(ghsa_ratelimit_remaining_sum[5m])` for simple last-value graphs. +- Grafana: stack remaining + used to visualise total limit per resource. diff --git a/docs/security/audit-events.md b/docs/security/audit-events.md new file mode 100644 index 00000000..5fcb16a9 --- /dev/null +++ b/docs/security/audit-events.md @@ -0,0 +1,47 @@ +# StellaOps Authority Audit Events + +StellaOps Authority emits structured audit records for every credential flow and bootstrap operation. The goal is to provide deterministic, privacy-aware telemetry that can be persisted offline and replayed for incident response without leaking credentials. + +## Contract + +Audit events share the `StellaOps.Cryptography.Audit.AuthEventRecord` contract. Key fields: + +- `EventType` — canonical identifier such as `authority.password.grant`, `authority.client_credentials.grant`, or `authority.bootstrap.user`. +- `OccurredAt` — UTC timestamp captured at emission time. +- `CorrelationId` — stable identifier propagated across logs and persistence. +- `Outcome` — `Success`, `Failure`, `LockedOut`, `RateLimited`, or `Error`. +- `Reason` — optional failure or policy message. +- `Subject` — `AuthEventSubject` carrying subject identifier, username, display name, and optional realm metadata. All subject fields are tagged as PII. +- `Client` — `AuthEventClient` with client identifier, display name, and originating provider/plugin. +- `Scopes` — granted or requested OAuth scopes (sorted before emission). +- `Network` — `AuthEventNetwork` with remote address, forwarded headers, and user agent string (all treated as PII). +- `Properties` — additional `AuthEventProperty` entries for context-specific details (lockout durations, policy decisions, retries, etc.). + +## Data Classifications + +Every string value uses `ClassifiedString` to assign a data classification: + +- `None` — public or operational metadata (event type, outcome). +- `Personal` — personally identifiable information (PII) such as subject identifiers, usernames, remote IP addresses, and user agents. +- `Sensitive` — secrets or derived credentials (client secrets, retry tokens). Avoid storing raw credentials; emit only hashed or summarised data when the classification is `Sensitive`. + +Downstream log sinks and persistence layers can inspect classifications to redact or separate PII before export. + +## Event Naming + +Event names follow dotted notation: + +- `authority.password.grant` — password grant handled by OpenIddict. +- `authority.client_credentials.grant` — client credential grant handling. +- `authority.bootstrap.user` and `authority.bootstrap.client` — bootstrap API operations. +- Future additions should preserve the `authority..` pattern to keep filtering deterministic. + +## Persistence + +The Authority host converts audit records into `AuthorityLoginAttemptDocument` rows for MongoDB persistence. Documents must: + +- Preserve `CorrelationId`, `SubjectId`, `ClientId`, `Plugin`, `Outcome`, `Reason`, and `OccurredAt`. +- Store remote address in `remoteAddress` only after classification as PII. +- Include summary booleans such as `Successful` to accelerate lockout policy checks. + +When exporting to external SIEMs, honour the `ClassifiedString.Classification` tag to avoid shipping PII into restricted environments. diff --git a/docs/security/authority-threat-model.md b/docs/security/authority-threat-model.md new file mode 100644 index 00000000..9cbba654 --- /dev/null +++ b/docs/security/authority-threat-model.md @@ -0,0 +1,106 @@ +# Authority Threat Model (STRIDE) + +> Prepared by Security Guild — 2025-10-12. Scope covers Authority host, Standard plug-in, CLI, bootstrap workflow, and offline revocation distribution. + +## 1. Scope & Method + +- Methodology: STRIDE applied to primary Authority surfaces (token issuance, bootstrap, revocation, operator tooling, plug-in extensibility). +- Assets in scope: identity credentials, OAuth tokens (access/refresh), bootstrap invites, revocation manifests, signing keys, audit telemetry. +- Out of scope: Third-party IdPs federated via OpenIddict (tracked separately in SEC6 backlog). + +## 2. Assets & Entry Points + +| Asset / Surface | Description | Primary Actors | +|-----------------|-------------|----------------| +| Token issuance APIs (`/token`, `/authorize`) | OAuth/OIDC endpoints mediated by OpenIddict | CLI, UI, automation agents | +| Bootstrap channel | Initial admin invite + bootstrap CLI workflow | Platform operators | +| Revocation bundle | Offline JSON + detached JWS consumed by agents | Feedser, Agents, Zastava | +| Plug-in manifests | Standard plug-in configuration and password policy overrides | Operators, DevOps | +| Signing keys | ES256 signing keys backing tokens and revocation manifests | Security Guild, HSM/KeyOps | +| Audit telemetry | Structured login/audit stream persisted to Mongo/observability stack | SOC, SecOps | + +## 3. Trust Boundaries + +| Boundary | Rationale | Controls | +|----------|-----------|----------| +| TB1 — Public network ↔️ Authority ingress | Internet/extranet exposure for `/token`, `/authorize`, `/bootstrap` | TLS 1.3, reverse proxy ACLs, rate limiting (SEC3.A / CORE8.RL) | +| TB2 — Authority host ↔️ Mongo storage | Credential store, revocation state, audit log persistence | Authenticated Mongo, network segmentation, deterministic serializers | +| TB3 — Authority host ↔️ Plug-in sandbox | Plug-ins may override password policy and bootstrap flows | Code signing, manifest validation, restart-time loading only | +| TB4 — Operator workstation ↔️ CLI | CLI holds bootstrap secrets and revocation bundles | OS keychain storage, MFA on workstations, offline kit checksum | +| TB5 — Authority ↔️ Downstream agents | Revocation bundle consumption, token validation | Mutual TLS (planned), detached JWS signatures, bundle freshness checks | + +## 4. Data Flow Diagrams + +### 4.1 Runtime token issuance + +```mermaid +flowchart LR + subgraph Client Tier + CLI[StellaOps CLI] + UI[UI / Automation] + end + subgraph Perimeter + RP[Reverse Proxy / WAF] + end + subgraph Authority + AUTH[Authority Host] + PLGIN[Standard Plug-in] + STORE[(Mongo Credential Store)] + end + CLI -->|OAuth password / client creds| RP --> AUTH + UI -->|OAuth flows| RP + AUTH -->|PasswordHashOptions + Secrets| PLGIN + AUTH -->|Verify / Persist hashes| STORE + STORE -->|Rehash needed| AUTH + AUTH -->|Access / refresh token| RP --> Client Tier +``` + +### 4.2 Bootstrap & revocation + +```mermaid +flowchart LR + subgraph Operator + OPS[Operator Workstation] + end + subgraph Authority + AUTH[Authority Host] + STORE[(Mongo)] + end + subgraph Distribution + OFFKIT[Offline Kit Bundle] + AGENT[Authorized Agent / Feedser] + end + OPS -->|Bootstrap CLI (`stellaops auth bootstrap`)| AUTH + AUTH -->|One-time invite + Argon2 hash| STORE + AUTH -->|Revocation export (`stellaops auth revoke export`)| OFFKIT + OFFKIT -->|Signed JSON + .jws| AGENT + AGENT -->|Revocation ACK / telemetry| AUTH +``` + +## 5. STRIDE Analysis + +| Threat | STRIDE Vector | Surface | Risk (L×I) | Existing Controls | Gaps / Actions | Owner | +|--------|---------------|---------|------------|-------------------|----------------|-------| +| Spoofed revocation bundle | Spoofing | TB5 — Authority ↔️ Agents | Med×High | Detached JWS signature (planned), offline kit checksums | Finalise signing key registry & verification script (SEC4.B/SEC4.HOST); add bundle freshness requirement | Security Guild (follow-up: **SEC5.B**) | +| Parameter tampering on `/token` | Tampering | TB1 — Public ingress | Med×High | ASP.NET model validation, OpenIddict, rate limiter (CORE8.RL) | Add audit coverage for tampered inputs, align correlation IDs with SOC (SEC2.A/SEC2.B) | Security Guild + Authority Core (follow-up: **SEC5.C**) | +| Bootstrap invite replay | Repudiation | TB4 — Operator CLI ↔️ Authority | Low×High | One-time bootstrap tokens, Argon2id hashing on creation | Enforce invite expiration + audit trail for unused invites | Security Guild (follow-up: **SEC5.D**) | +| Token replay by stolen agent | Information Disclosure | TB5 | Med×High | Planned revocation bundles, optional mTLS | Require agent binding (device fingerprint) and enforce revocation grace window alerts | Security Guild + Zastava (follow-up: **SEC5.E**) | +| Privilege escalation via plug-in override | Elevation of Privilege | TB3 — Plug-in sandbox | Med×High | Signed plug-ins, restart-only loading, configuration validation | Add static analysis on manifest overrides + runtime warning when policy weaker than host | Security Guild + DevOps (follow-up: **SEC5.F**) | +| Offline bundle tampering | Tampering | Distribution | Low×High | SHA256 manifest, signed bundles (planned) | Add supply-chain attestation for Offline Kit, publish verification CLI in docs | Security Guild + Ops (follow-up: **SEC5.G**) | +| Failure to log denied tokens | Repudiation | TB2 — Authority ↔️ Mongo | Med×Med | Serilog structured events (partial), Mongo persistence path (planned) | Finalise audit schema (SEC2.A) and ensure `/token` denies include subject/client/IP fields | Security Guild + Authority Core (follow-up: **SEC5.H**) | + +Risk scoring uses qualitative scale (Low/Med/High) for likelihood × impact; mitigation priority follows High > Med > Low. + +## 6. Follow-up Backlog Hooks + +| Backlog ID | Linked Threat | Summary | Target Owners | +|------------|---------------|---------|---------------| +| SEC5.B | Spoofed revocation bundle | Complete libsodium/Core signing integration and ship revocation verification script. | Security Guild + Authority Core | +| SEC5.C | Parameter tampering on `/token` | Finalise audit contract (`SEC2.A`) and add request tamper logging. | Security Guild + Authority Core | +| SEC5.D | Bootstrap invite replay | Implement expiry enforcement + audit coverage for unused bootstrap invites. | Security Guild | +| SEC5.E | Token replay by stolen agent | Document device binding requirements and create detector for stale revocation acknowledgements. | Security Guild + Zastava | +| SEC5.F | Plug-in override escalation | Static analysis of plug-in manifests; warn on weaker password policy overrides. | Security Guild + DevOps | +| SEC5.G | Offline bundle tampering | Extend Offline Kit build to include attested manifest + verification CLI sample. | Security Guild + Ops | +| SEC5.H | Failure to log denied tokens | Ensure audit persistence for all `/token` denials with correlation IDs. | Security Guild + Authority Core | + +Update `src/StellaOps.Cryptography/TASKS.md` (Security Guild board) with the above backlog entries to satisfy SEC5.A exit criteria. diff --git a/docs/security/password-hashing.md b/docs/security/password-hashing.md new file mode 100644 index 00000000..9ae3fb46 --- /dev/null +++ b/docs/security/password-hashing.md @@ -0,0 +1,76 @@ +# Authority Password Hashing Guidance + +> **Status:** Drafted 2025-10-11 alongside SEC1.A / SEC1.PLG rollout. Argon2id is now the default hashing algorithm for the Standard plug-in and recommended for all Authority identity providers. + +## 1. Overview + +StellaOps Authority issues and verifies credentials through the shared `StellaOps.Cryptography` provider abstraction. As of October 2025: + +- **Default algorithm:** Argon2id (PHC format `$argon2id$v=19$m=,t=