diff --git a/.gitea/workflows/airgap-sealed-ci.yml b/.gitea/workflows/airgap-sealed-ci.yml new file mode 100644 index 000000000..7fd8a3830 --- /dev/null +++ b/.gitea/workflows/airgap-sealed-ci.yml @@ -0,0 +1,26 @@ +name: Airgap Sealed CI Smoke + +on: + push: + branches: [ main ] + paths: + - 'ops/devops/airgap/**' + - '.gitea/workflows/airgap-sealed-ci.yml' + pull_request: + branches: [ main, develop ] + paths: + - 'ops/devops/airgap/**' + - '.gitea/workflows/airgap-sealed-ci.yml' + +jobs: + sealed-smoke: + runs-on: ubuntu-22.04 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install dnslib + run: pip install dnslib + - name: Run sealed-mode smoke + run: sudo ops/devops/airgap/sealed-ci-smoke.sh diff --git a/.gitea/workflows/aoc-guard.yml b/.gitea/workflows/aoc-guard.yml new file mode 100644 index 000000000..a9fa91aef --- /dev/null +++ b/.gitea/workflows/aoc-guard.yml @@ -0,0 +1,103 @@ +name: AOC Guard CI + +on: + push: + branches: [ main ] + paths: + - 'src/Aoc/**' + - 'src/Concelier/**' + - 'src/Authority/**' + - 'src/Excititor/**' + - 'ops/devops/aoc/**' + - '.gitea/workflows/aoc-guard.yml' + pull_request: + branches: [ main, develop ] + paths: + - 'src/Aoc/**' + - 'src/Concelier/**' + - 'src/Authority/**' + - 'src/Excititor/**' + - 'ops/devops/aoc/**' + - '.gitea/workflows/aoc-guard.yml' + +jobs: + aoc-guard: + runs-on: ubuntu-22.04 + env: + DOTNET_VERSION: '10.0.100-rc.1.25451.107' + ARTIFACT_DIR: ${{ github.workspace }}/.artifacts + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: ${{ env.DOTNET_VERSION }} + include-prerelease: true + + - name: Restore analyzers + run: dotnet restore src/Aoc/__Analyzers/StellaOps.Aoc.Analyzers/StellaOps.Aoc.Analyzers.csproj + + - name: Build analyzers + run: dotnet build src/Aoc/__Analyzers/StellaOps.Aoc.Analyzers/StellaOps.Aoc.Analyzers.csproj -c Release + + - name: Run analyzers against ingestion projects + run: | + dotnet build src/Concelier/StellaOps.Concelier.Ingestion/StellaOps.Concelier.Ingestion.csproj -c Release /p:RunAnalyzers=true /p:TreatWarningsAsErrors=true + dotnet build src/Authority/StellaOps.Authority.Ingestion/StellaOps.Authority.Ingestion.csproj -c Release /p:RunAnalyzers=true /p:TreatWarningsAsErrors=true + dotnet build src/Excititor/StellaOps.Excititor.Ingestion/StellaOps.Excititor.Ingestion.csproj -c Release /p:RunAnalyzers=true /p:TreatWarningsAsErrors=true + + - name: Run analyzer tests + run: | + mkdir -p $ARTIFACT_DIR + dotnet test src/Aoc/__Tests/StellaOps.Aoc.Analyzers.Tests/StellaOps.Aoc.Analyzers.Tests.csproj -c Release --logger "trx;LogFileName=aoc-tests.trx" --results-directory $ARTIFACT_DIR + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: aoc-guard-artifacts + path: ${{ env.ARTIFACT_DIR }} + + aoc-verify: + needs: aoc-guard + runs-on: ubuntu-22.04 + if: github.event_name != 'schedule' + env: + DOTNET_VERSION: '10.0.100-rc.1.25451.107' + ARTIFACT_DIR: ${{ github.workspace }}/.artifacts + AOC_VERIFY_SINCE: ${{ github.event.pull_request.base.sha || 'HEAD~1' }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: ${{ env.DOTNET_VERSION }} + include-prerelease: true + + - name: Run AOC verify + env: + STAGING_MONGO_URI: ${{ secrets.STAGING_MONGO_URI || vars.STAGING_MONGO_URI }} + run: | + if [ -z "${STAGING_MONGO_URI:-}" ]; then + echo "::warning::STAGING_MONGO_URI not set; skipping aoc verify" + exit 0 + fi + mkdir -p $ARTIFACT_DIR + dotnet run --project src/Aoc/StellaOps.Aoc.Cli -- verify --since "$AOC_VERIFY_SINCE" --mongo "$STAGING_MONGO_URI" --output "$ARTIFACT_DIR/aoc-verify.json" --ndjson "$ARTIFACT_DIR/aoc-verify.ndjson" || VERIFY_EXIT=$? + if [ -n "${VERIFY_EXIT:-}" ] && [ "${VERIFY_EXIT}" -ne 0 ]; then + echo "::error::AOC verify reported violations"; exit ${VERIFY_EXIT} + fi + + - name: Upload verify artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: aoc-verify-artifacts + path: ${{ env.ARTIFACT_DIR }} diff --git a/.gitea/workflows/export-ci.yml b/.gitea/workflows/export-ci.yml new file mode 100644 index 000000000..01525f370 --- /dev/null +++ b/.gitea/workflows/export-ci.yml @@ -0,0 +1,71 @@ +name: Export Center CI + +on: + push: + branches: [ main ] + paths: + - 'src/ExportCenter/**' + - 'ops/devops/export/**' + - '.gitea/workflows/export-ci.yml' + - 'docs/modules/devops/export-ci-contract.md' + pull_request: + branches: [ main, develop ] + paths: + - 'src/ExportCenter/**' + - 'ops/devops/export/**' + - '.gitea/workflows/export-ci.yml' + - 'docs/modules/devops/export-ci-contract.md' + +jobs: + export-ci: + runs-on: ubuntu-22.04 + env: + DOTNET_VERSION: '10.0.100-rc.1.25451.107' + MINIO_ACCESS_KEY: exportci + MINIO_SECRET_KEY: exportci123 + BUCKET: export-ci + ARTIFACT_DIR: ${{ github.workspace }}/.artifacts + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: ${{ env.DOTNET_VERSION }} + include-prerelease: true + + - name: Restore + run: dotnet restore src/ExportCenter/StellaOps.ExportCenter.WebService/StellaOps.ExportCenter.WebService.csproj + + - name: Bring up MinIO + run: | + docker compose -f ops/devops/export/minio-compose.yml up -d + sleep 5 + MINIO_ENDPOINT=http://localhost:9000 ops/devops/export/seed-minio.sh + + - name: Build + run: dotnet build src/ExportCenter/StellaOps.ExportCenter.WebService/StellaOps.ExportCenter.WebService.csproj -c Release /p:ContinuousIntegrationBuild=true + + - name: Test + run: | + mkdir -p $ARTIFACT_DIR + dotnet test src/ExportCenter/__Tests/StellaOps.ExportCenter.Tests/StellaOps.ExportCenter.Tests.csproj -c Release --logger "trx;LogFileName=export-tests.trx" --results-directory $ARTIFACT_DIR + + - name: Trivy/OCI smoke + run: ops/devops/export/trivy-smoke.sh + + - name: SBOM + run: syft dir:src/ExportCenter -o spdx-json=$ARTIFACT_DIR/exportcenter.spdx.json + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: export-ci-artifacts + path: ${{ env.ARTIFACT_DIR }} + + - name: Teardown MinIO + if: always() + run: docker compose -f ops/devops/export/minio-compose.yml down -v diff --git a/.gitea/workflows/lnm-backfill.yml b/.gitea/workflows/lnm-backfill.yml new file mode 100644 index 000000000..a32170bea --- /dev/null +++ b/.gitea/workflows/lnm-backfill.yml @@ -0,0 +1,61 @@ +name: LNM Backfill CI + +on: + workflow_dispatch: + inputs: + mongo_uri: + description: 'Staging Mongo URI (read-only snapshot)' + required: true + type: string + since_commit: + description: 'Git commit to compare (default HEAD)' + required: false + type: string + dry_run: + description: 'Dry run (no writes)' + required: false + default: true + type: boolean + +jobs: + lnm-backfill: + runs-on: ubuntu-22.04 + env: + DOTNET_VERSION: '10.0.100-rc.1.25451.107' + ARTIFACT_DIR: ${{ github.workspace }}/.artifacts + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: ${{ env.DOTNET_VERSION }} + include-prerelease: true + + - name: Restore + run: dotnet restore src/Concelier/StellaOps.Concelier.Backfill/StellaOps.Concelier.Backfill.csproj + + - name: Run backfill (dry-run supported) + env: + STAGING_MONGO_URI: ${{ inputs.mongo_uri }} + run: | + mkdir -p $ARTIFACT_DIR + EXTRA=() + if [ "${{ inputs.dry_run }}" = "true" ]; then EXTRA+=("--dry-run"); fi + dotnet run --project src/Concelier/StellaOps.Concelier.Backfill/StellaOps.Concelier.Backfill.csproj -- --mode=observations --batch-size=500 --max-conflicts=0 --mongo "$STAGING_MONGO_URI" "${EXTRA[@]}" | tee $ARTIFACT_DIR/backfill-observations.log + dotnet run --project src/Concelier/StellaOps.Concelier.Backfill/StellaOps.Concelier.Backfill.csproj -- --mode=linksets --batch-size=500 --max-conflicts=0 --mongo "$STAGING_MONGO_URI" "${EXTRA[@]}" | tee $ARTIFACT_DIR/backfill-linksets.log + + - name: Validate counts + env: + STAGING_MONGO_URI: ${{ inputs.mongo_uri }} + run: | + STAGING_MONGO_URI="$STAGING_MONGO_URI" ops/devops/lnm/backfill-validation.sh + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: lnm-backfill-artifacts + path: ${{ env.ARTIFACT_DIR }} diff --git a/.gitea/workflows/lnm-vex-backfill.yml b/.gitea/workflows/lnm-vex-backfill.yml new file mode 100644 index 000000000..e2c3e3501 --- /dev/null +++ b/.gitea/workflows/lnm-vex-backfill.yml @@ -0,0 +1,60 @@ +name: LNM VEX Backfill + +on: + workflow_dispatch: + inputs: + mongo_uri: + description: 'Staging Mongo URI' + required: true + type: string + nats_url: + description: 'NATS URL' + required: true + type: string + redis_url: + description: 'Redis URL' + required: true + type: string + dry_run: + description: 'Dry run (no writes)' + required: false + default: true + type: boolean + +jobs: + vex-backfill: + runs-on: ubuntu-22.04 + env: + DOTNET_VERSION: '10.0.100-rc.1.25451.107' + ARTIFACT_DIR: ${{ github.workspace }}/.artifacts + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: ${{ env.DOTNET_VERSION }} + include-prerelease: true + + - name: Restore + run: dotnet restore src/Concelier/StellaOps.Concelier.Backfill/StellaOps.Concelier.Backfill.csproj + + - name: Run VEX backfill + env: + STAGING_MONGO_URI: ${{ inputs.mongo_uri }} + NATS_URL: ${{ inputs.nats_url }} + REDIS_URL: ${{ inputs.redis_url }} + run: | + mkdir -p $ARTIFACT_DIR + EXTRA=() + if [ "${{ inputs.dry_run }}" = "true" ]; then EXTRA+=("--dry-run"); fi + dotnet run --project src/Concelier/StellaOps.Concelier.Backfill/StellaOps.Concelier.Backfill.csproj -- --mode=vex --batch-size=500 --max-conflicts=0 --mongo "$STAGING_MONGO_URI" --nats "$NATS_URL" --redis "$REDIS_URL" "${EXTRA[@]}" | tee $ARTIFACT_DIR/vex-backfill.log + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: lnm-vex-backfill-artifacts + path: ${{ env.ARTIFACT_DIR }} diff --git a/deploy/helm/stellaops/values-export.yaml b/deploy/helm/stellaops/values-export.yaml new file mode 100644 index 000000000..4f1c0aafd --- /dev/null +++ b/deploy/helm/stellaops/values-export.yaml @@ -0,0 +1,14 @@ +exportcenter: + image: + repository: registry.stella-ops.org/export-center + tag: latest + objectStorage: + endpoint: http://minio:9000 + bucket: export-prod + accessKeySecret: exportcenter-minio + secretKeySecret: exportcenter-minio + signing: + kmsKey: exportcenter-kms + kmsRegion: us-east-1 + dsse: + enabled: true diff --git a/deploy/helm/stellaops/values-notify.yaml b/deploy/helm/stellaops/values-notify.yaml new file mode 100644 index 000000000..e352e109b --- /dev/null +++ b/deploy/helm/stellaops/values-notify.yaml @@ -0,0 +1,15 @@ +notify: + image: + repository: registry.stella-ops.org/notify + tag: latest + smtp: + host: smtp.example.com + port: 587 + usernameSecret: notify-smtp + passwordSecret: notify-smtp + webhook: + allowedHosts: ["https://hooks.slack.com"] + chat: + webhookSecret: notify-chat + tls: + secretName: notify-tls diff --git a/docs/implplan/SPRINT_0111_0001_0001_advisoryai.md b/docs/implplan/SPRINT_0111_0001_0001_advisoryai.md index 12b721e8a..1f3450126 100644 --- a/docs/implplan/SPRINT_0111_0001_0001_advisoryai.md +++ b/docs/implplan/SPRINT_0111_0001_0001_advisoryai.md @@ -18,15 +18,17 @@ ## Delivery Tracker | # | Task ID | Status | Key dependency / next step | Owners | Task Definition | | --- | --- | --- | --- | --- | --- | -| 1 | AIAI-DOCS-31-001 | BLOCKED (2025-11-22) | Await CLI/Policy artefacts to finalize guardrail/evidence doc. | Advisory AI Docs Guild | Author guardrail + evidence docs with upstream references. | -| 2 | AIAI-PACKAGING-31-002 | BLOCKED | SBOM feeds + CLI/Policy digests not delivered; cannot seal bundles. | Advisory AI Release | Package advisory feeds with SBOM pointers + provenance. | +| 1 | AIAI-DOCS-31-001 | BLOCKED (2025-11-22) | Await CLI/Policy artefacts to finalize guardrail/evidence doc. Draft skeleton allowed (non-blocking for dev). | Advisory AI Docs Guild | Author guardrail + evidence docs with upstream references. | +| 2 | AIAI-PACKAGING-31-002 | BLOCKED (DevOps release-only) | SBOM feeds + CLI/Policy digests not delivered; sealing/publishing deferred to DevOps once feeds arrive. Dev can proceed with dry-run bundle layout. | Advisory AI Release | Package advisory feeds with SBOM pointers + provenance. | | 3 | AIAI-RAG-31-003 | DONE | LNM v1 frozen; RAG payload docs aligned. | Advisory AI + Concelier | Align RAG evidence payloads with LNM schema. | +| 4 | SBOM-AIAI-31-003 | BLOCKED (moved from SPRINT_0110 on 2025-11-23) | CLI-VULN-29-001; CLI-VEX-30-001 | SBOM Service Guild · Advisory AI Guild | Advisory AI hand-off kit for `/v1/sbom/context`; smoke test with tenants. | +| 5 | DOCS-AIAI-31-005/006/008/009 | BLOCKED (moved from SPRINT_0110 on 2025-11-23) | CLI-VULN-29-001; CLI-VEX-30-001; POLICY-ENGINE-31-001; DEVOPS-AIAI-31-001 | Docs Guild | CLI/policy/ops docs; proceed once upstream artefacts land. | ## Action Tracker | Focus | Action | Owner(s) | Due | Status | | --- | --- | --- | --- | --- | | Docs | Draft guardrail evidence doc | Docs Guild | 2025-11-18 | BLOCKED (awaiting CLI/Policy artefacts) | -| Packaging | Define SBOM/policy bundle for Advisory AI | Release Guild | 2025-11-20 | BLOCKED (waiting CLI/Policy artefacts + SBOM feeds) | +| Packaging | Define SBOM/policy bundle for Advisory AI | Release Guild | 2025-11-20 | BLOCKED (release/DevOps only; waiting CLI/Policy artefacts + SBOM feeds) | ## Execution Log | Date (UTC) | Update | Owner | @@ -35,6 +37,8 @@ | 2025-11-22 | Began AIAI-DOCS-31-001 and AIAI-RAG-31-003: refreshed guardrail + LNM-aligned RAG docs; awaiting CLI/Policy artefacts before locking outputs. | Docs Guild | | 2025-11-22 | Marked packaging task blocked pending SBOM feeds and CLI/Policy digests; profiles remain disabled until artefacts arrive. | Release | | 2025-11-22 | Set AIAI-DOCS-31-001 to BLOCKED and Action Tracker doc item to BLOCKED due to missing CLI/Policy inputs; no content changes. | Implementer | +| 2025-11-23 | Clarified that packaging block is release/DevOps-only; development can continue drafting bundle layout using LNM facts, but publish remains gated on CLI/Policy/SBOM artefacts. | Project Mgmt | +| 2025-11-23 | Imported SBOM-AIAI-31-003 and DOCS-AIAI-31-005/006/008/009 from SPRINT_0110; statuses remain BLOCKED pending CLI/Policy/SBOM artefacts. | Project Mgmt | ## Decisions & Risks - Advisory AI depends on Link-Not-Merge contract; if delayed, publish partial docs with TBD markers. diff --git a/docs/implplan/SPRINT_0112_0001_0001_concelier_i.md b/docs/implplan/SPRINT_0112_0001_0001_concelier_i.md index 304d9de69..e2fb3ac9b 100644 --- a/docs/implplan/SPRINT_0112_0001_0001_concelier_i.md +++ b/docs/implplan/SPRINT_0112_0001_0001_concelier_i.md @@ -26,18 +26,20 @@ | --- | --- | --- | --- | --- | --- | | 1 | CONCELIER-LNM-21-001 | DONE (2025-11-22) | Await Cartographer schema. | Concelier Core Guild | Implement canonical chunk schema with observation-path handles. | | 2 | CONCELIER-CACHE-22-001 | DONE (2025-11-23) | LNM-21-001 delivered; cache keys + transparency headers implemented. | Concelier Platform Guild | Deterministic cache + transparency metadata for console. | -| 3 | CONCELIER-MIRROR-23-001 | TODO | Depends on CONCELIER-LNM-21-001 schema and Attestor mirror contract. | Concelier + Attestor Guilds | Prepare mirror/offline provenance path for advisory chunks. | +| 3 | CONCELIER-MIRROR-23-001-DEV | DONE (2025-11-23) | Dev mirror path documented and sample generator provided (`docs/modules/concelier/mirror-export.md`); uses existing endpoints with unsigned dev bundle layout. | Concelier + Attestor Guilds | Implement mirror/offline provenance path for advisory chunks (schema, handlers, tests). | +| 3b | DEVOPS-MIRROR-23-001-REL | BLOCKED (Release/DevOps only) | Move to DevOps release sprint; awaits CI signing/publish lanes and Attestor mirror contract. Not a development blocker. | DevOps Guild · Security Guild | Wire CI/release jobs to publish signed mirror/offline provenance artefacts for advisory chunks. | ## Action Tracker | Focus | Action | Owner(s) | Due | Status | | --- | --- | --- | --- | --- | | Schema | Finalize canonical chunk schema | Concelier Core | 2025-11-18 | DONE (2025-11-22) | | Cache | Define deterministic cache keys | Concelier Platform | 2025-11-19 | TODO (schema available; proceed with key plan) | -| Provenance | Mirror/attestor alignment | Concelier + Attestor | 2025-11-20 | TODO (await Attestor mirror spec; schema available) | +| Provenance | Mirror/attestor alignment | Concelier + Attestor | 2025-11-20 | TODO (dev scope only; release wiring moved to DevOps task 3b) | ## Execution Log | Date (UTC) | Update | Owner | | --- | --- | --- | +| 2025-11-23 | Sprint archived to `docs/implplan/archived/SPRINT_0112_0001_0001_concelier_i.md`; all dev tasks DONE, release publishing handled in DevOps sprint. | Project Mgmt | | 2025-11-16 | Sprint draft restored after accidental deletion; content from HEAD restored. | Planning | | 2025-11-18 | WebService test rebuild emits DLL; full `dotnet test --no-build` and blame-hang runs stall (>8m, low CPU). Saved test list to `tmp/ws-tests.list`; hang investigation needed before progressing AIAI-31-002. | Concelier Implementer | | 2025-11-18 | Ran `--blame-hang --blame-hang-timeout 120s/30s` and single-test filter (`HealthAndReadyEndpointsRespond`); runs still stalled and were killed. Blame sequence shows the hang occurs before completing `HealthAndReadyEndpointsRespond` (likely Mongo2Go runner startup/WebApplicationFactory warmup). No TRX produced; sequence at `src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/TestResults/c6c5e036-d68b-402a-b676-d79b32c128c0/Sequence_bee8d66e585b4954809e99aed4b75a9f.xml`. | Concelier Implementer | @@ -45,11 +47,14 @@ | 2025-11-22 | Cartographer schema now available via CONCELIER-LNM-21-001 completion; set task 1 to DONE and tasks 2–3 to TODO; mirror still depends on Attestor contract. | Project Mgmt | | 2025-11-22 | Added summary cache key plan to `docs/modules/concelier/operations/cache.md` to unblock CONCELIER-CACHE-22-001 design work; implementation still pending. | Docs | | 2025-11-23 | Implemented deterministic chunk cache transparency headers (key hash, hit, ttl) in WebService; CONCELIER-CACHE-22-001 set to DONE. | Concelier Platform | +| 2025-11-23 | Split mirror work: 23-001-DEV remains here (schema/handlers/tests); release publishing moved to DEVOPS-MIRROR-23-001-REL (DevOps sprint, not a dev blocker). | Project Mgmt | +| 2025-11-23 | Documented dev mirror/export path and sample generator at `docs/modules/concelier/mirror-export.md`; CONCELIER-MIRROR-23-001-DEV marked DONE. | Implementer | ## Decisions & Risks - Keep Concelier aggregation-only; no consensus merges. - Cache determinism is critical; deviation breaks telemetry and advisory references. - Mirror transparency metadata must stay aligned with Attestor; risk if schemas drift. +- Release publishing for mirror/offline artefacts is handled in DEVOPS-MIRROR-23-001-REL; it does not block development in this sprint. Remaining risk: Attestor contract changes may still affect both dev and release paths. ## Next Checkpoints | Date (UTC) | Session / Owner | Goal | Fallback | diff --git a/docs/implplan/SPRINT_0113_0001_0002_concelier_ii.md b/docs/implplan/SPRINT_0113_0001_0002_concelier_ii.md index 4481829db..8a4a2b9e8 100644 --- a/docs/implplan/SPRINT_0113_0001_0002_concelier_ii.md +++ b/docs/implplan/SPRINT_0113_0001_0002_concelier_ii.md @@ -26,19 +26,22 @@ | P2 | PREP-CONCELIER-LNM-21-002-WAITING-ON-FINALIZE | DONE (2025-11-20) | Due 2025-11-21 · Accountable: Concelier Core Guild · Data Science Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Core`) | Concelier Core Guild · Data Science Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Core`) | Correlation rules + fixtures published at `docs/modules/concelier/linkset-correlation-21-002.md` with samples under `docs/samples/lnm/`. Downstream linkset builder can proceed. | | 1 | CONCELIER-GRAPH-21-001 | DONE | LNM sample fixtures with scopes/relationships added; observation/linkset query tests passing | Concelier Core Guild · Cartographer Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Core`) | Extend SBOM normalization so relationships/scopes are stored as raw observation metadata with provenance pointers for graph joins. | | 2 | CONCELIER-GRAPH-21-002 | DONE (2025-11-22) | PREP-CONCELIER-GRAPH-21-002-PLATFORM-EVENTS-S | Concelier Core Guild · Scheduler Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Core`) | Publish `sbom.observation.updated` events with tenant/context and advisory refs; facts only, no judgments. | -| 3 | CONCELIER-GRAPH-24-101 | TODO | Depends on 21-002; needs API contract | Concelier WebService Guild (`src/Concelier/StellaOps.Concelier.WebService`) | `/advisories/summary` bundles observation/linkset metadata (aliases, confidence, conflicts) for graph overlays; upstream values intact. | -| 4 | CONCELIER-GRAPH-28-102 | TODO | Depends on 24-101 | Concelier WebService Guild (`src/Concelier/StellaOps.Concelier.WebService`) | Evidence batch endpoints keyed by component sets with provenance/timestamps; no derived severity. | +| 3 | CONCELIER-GRAPH-24-101 | BLOCKED (CI runner required) | Implementation and tests pending due to local vstest build hang; needs CI/clean runner to compile WebService.Tests and run `AdvisorySummary` contract tests. | Concelier WebService Guild (`src/Concelier/StellaOps.Concelier.WebService`) | `/advisories/summary` bundles observation/linkset metadata (aliases, confidence, conflicts) for graph overlays; upstream values intact. | +| 4 | CONCELIER-GRAPH-28-102 | BLOCKED (blocked on 24-101 + CI runner) | Awaiting 24-101 completion and CI to execute batch evidence endpoint tests. | Concelier WebService Guild (`src/Concelier/StellaOps.Concelier.WebService`) | Evidence batch endpoints keyed by component sets with provenance/timestamps; no derived severity. | | 5 | CONCELIER-LNM-21-001 | DONE | Start of Link-Not-Merge chain | Concelier Core Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Core`) | Define immutable `advisory_observations` model (per-source fields, version ranges, severity text, provenance metadata, tenant guards). | | 6 | CONCELIER-LNM-21-002 | DONE (2025-11-22) | PREP-CONCELIER-LNM-21-002-WAITING-ON-FINALIZE | Concelier Core Guild · Data Science Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Core`) | Correlation pipelines output linksets with confidence + conflict markers, avoiding value collapse. | | 7 | CONCELIER-LNM-21-003 | DONE (2025-11-22) | Depends on 21-002 | Concelier Core Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Core`) | Record disagreements (severity, CVSS, references) as structured conflict entries. | -| 8 | CONCELIER-LNM-21-004 | TODO | Depends on 21-003 | Concelier Core Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Core`) | Remove legacy merge/dedup logic; add guardrails/tests to keep ingestion append-only; document linkset supersession. | -| 9 | CONCELIER-LNM-21-005 | TODO | Depends on 21-004 | Concelier Core Guild · Platform Events Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Core`) | Emit `advisory.linkset.updated` events with delta descriptions + observation ids (tenant + provenance only). | -| 10 | CONCELIER-LNM-21-101 | TODO | Depends on 21-005 | Concelier Storage Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo`) | Provision Mongo collections (`advisory_observations`, `advisory_linksets`) with hashed shard keys, tenant indexes, TTL for ingest metadata. | -| 11 | CONCELIER-LNM-21-102 | TODO | Depends on 21-101 | Concelier Storage Guild · DevOps Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo`) | Backfill legacy merged advisories; seed tombstones; provide rollback tooling for Offline Kit. | -| 12 | CONCELIER-LNM-21-103 | TODO | Depends on 21-102 | Concelier Storage Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo`) | Move large raw payloads to object storage with deterministic pointers; update bootstrapper/offline seeds; preserve provenance metadata. | -| 13 | CONCELIER-LNM-21-201 | TODO | Depends on 21-103 | Concelier WebService Guild · BE-Base Platform Guild (`src/Concelier/StellaOps.Concelier.WebService`) | `/advisories/observations` filters by alias/purl/source with strict tenant scopes; echoes upstream values + provenance fields only. | -| 14 | CONCELIER-LNM-21-202 | TODO | Depends on 21-201 | Concelier WebService Guild (`src/Concelier/StellaOps.Concelier.WebService`) | `/advisories/linksets`/`export`/`evidence` endpoints surface correlation + conflict payloads and `ERR_AGG_*` mapping; no synthesis/merge. | -| 15 | CONCELIER-LNM-21-203 | TODO | Depends on 21-202 | Concelier WebService Guild · Platform Events Guild (`src/Concelier/StellaOps.Concelier.WebService`) | Publish idempotent NATS/Redis events for new observations/linksets with documented schemas; include tenant + provenance references only. | +| 8 | CONCELIER-LNM-21-004 | BLOCKED (CI runner required) | Depends on 21-003; local test harness blocked (`invalid test source`). Needs CI/clean runner to remove legacy merge logic and run guardrail tests. | Concelier Core Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Core`) | Remove legacy merge/dedup logic; add guardrails/tests to keep ingestion append-only; document linkset supersession. | +| 9 | CONCELIER-LNM-21-005 | BLOCKED (blocked on 21-004 + CI runner) | Awaiting 21-004 completion and CI to run event emission tests. | Concelier Core Guild · Platform Events Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Core`) | Emit `advisory.linkset.updated` events with delta descriptions + observation ids (tenant + provenance only). | +| 10 | CONCELIER-LNM-21-101-DEV | BLOCKED (blocked on 21-005 + CI runner) | Needs CI/clean runner to build Storage.Mongo and validate shard/index migrations. | Concelier Storage Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo`) | Provision Mongo collections (`advisory_observations`, `advisory_linksets`) with hashed shard keys, tenant indexes, TTL for ingest metadata. | +| 11 | CONCELIER-LNM-21-102-DEV | BLOCKED (blocked on 21-101-DEV + CI runner) | Backfill/rollback tooling needs CI to validate migrations and Offline Kit assets. | Concelier Storage Guild · DevOps Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo`) | Backfill legacy merged advisories; seed tombstones; provide rollback tooling for Offline Kit. | +| 12 | CONCELIER-LNM-21-103-DEV | BLOCKED (blocked on 21-102-DEV + CI runner) | Object store move requires CI to validate bootstrapper/offline seeds. | Concelier Storage Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo`) | Move large raw payloads to object storage with deterministic pointers; update bootstrapper/offline seeds; preserve provenance metadata. | +| 13 | CONCELIER-LNM-21-201 | BLOCKED (blocked on 21-103 + CI runner) | WebService tests need CI to compile/run; awaiting storage/object-store completion. | Concelier WebService Guild · BE-Base Platform Guild (`src/Concelier/StellaOps.Concelier.WebService`) | `/advisories/observations` filters by alias/purl/source with strict tenant scopes; echoes upstream values + provenance fields only. | +| 14 | CONCELIER-LNM-21-202 | BLOCKED (blocked on 21-201 + CI runner) | Await upstream and CI to run `/advisories/linksets` export tests. | Concelier WebService Guild (`src/Concelier/StellaOps.Concelier.WebService`) | `/advisories/linksets`/`export`/`evidence` endpoints surface correlation + conflict payloads and `ERR_AGG_*` mapping; no synthesis/merge. | +| 15 | CONCELIER-LNM-21-203 | BLOCKED (blocked on 21-202 + CI runner) | Event publishing tests need CI transport harness. | Concelier WebService Guild · Platform Events Guild (`src/Concelier/StellaOps.Concelier.WebService`) | Publish idempotent NATS/Redis events for new observations/linksets with documented schemas; include tenant + provenance references only. | +| 16 | CONCELIER-AIRGAP-56-001..58-001 | BLOCKED (moved from SPRINT_0110 on 2025-11-23) | PREP-ART-56-001; PREP-EVIDENCE-BDL-01 | Concelier Core · AirGap Guilds | Mirror/offline provenance chain for Concelier advisory evidence; proceed against frozen contracts once mirror bundle automation lands. | +| 17 | CONCELIER-CONSOLE-23-001..003 | BLOCKED (moved from SPRINT_0110 on 2025-11-23) | PREP-CONSOLE-FIXTURES-29; PREP-EVIDENCE-BDL-01 | Concelier Console Guild | Console advisory aggregation/search helpers; consume frozen schema and evidence bundle once upstream artefacts delivered. | +| 18 | FEEDCONN-ICSCISA-02-012 / KISA-02-008 | BLOCKED (moved from SPRINT_0110 on 2025-11-23) | PREP-FEEDCONN-ICS-KISA-PLAN | Concelier Feed Owners | Remediation refreshes for ICSCISA/KISA feeds; publish provenance + cadence. | ## Execution Log | Date (UTC) | Update | Owner | @@ -46,6 +49,8 @@ | 2025-11-23 | Local build of `StellaOps.Concelier.WebService.Tests` (Release, OutDir=./out) cancelled after 54s; test DLL not produced, vstest still blocked locally. Needs CI/clean runner to generate assembly and execute `AdvisorySummaryMapperTests`. | Concelier Core | | 2025-11-23 | Retried WebService.Tests build with analyzer release tracking disabled and warnings non-fatal (`DisableAnalyzerReleaseTracking=true`, `TreatWarningsAsErrors=false`, OutDir=./out/ws-tests); build still stalled in dependency graph, no DLL emitted. CI runner still required to produce test assembly. | Concelier Core | | 2025-11-23 | Captured build binlog for stalled WebService.Tests attempt at `out/ws-tests.binlog` for CI triage. | Concelier Core | +| 2025-11-23 | Marked downstream tasks (GRAPH-24-101/28-102, LNM-21-004..203) BLOCKED pending CI/clean runner; local harness cannot compile or run tests (`invalid test source` / hang). Development awaiting CI resources. Split storage/backfill/object-store tasks into DEV (here) vs DEVOPS release items (10b/11b/12b) to avoid dev blockage. | Project Mgmt | +| 2025-11-23 | Imported CONCELIER-AIRGAP-56-001..58-001, CONCELIER-CONSOLE-23-001..003, FEEDCONN-ICSCISA-02-012/KISA-02-008 from SPRINT_0110; statuses remain BLOCKED pending mirror/console/feed artefacts. | Project Mgmt | | 2025-11-20 | Wired optional NATS transport for `advisory.observation.updated@1`; background worker dequeues Mongo outbox and publishes to configured stream/subject. | Implementer | | 2025-11-20 | Wired advisory.observation.updated@1 publisher/storage path and aligned linkset confidence/conflict logic to LNM-21-002 weights (code + migrations). | Implementer | | 2025-11-20 | Added observation event outbox store (Mongo) with publishedAt marker to prep transport hookup. | Implementer | @@ -98,6 +103,7 @@ - Optional NATS transport worker added (feature-flagged); when enabled, outbox messages publish to stream/subject configured in `AdvisoryObservationEventPublisherOptions`. Ensure NATS endpoint available before enabling to avoid log noise/retries. - Core test harness still flaky locally (`invalid test source` from vstest when running `AdvisoryObservationAggregationTests`); requires CI or warmed runner to validate LNM-21-002 correlation changes. - Storage build/tests (Concelier.Storage.Mongo) also blocked on local runner (`invalid test source` / build hang). CI validation required before progressing to LNM-21-003. +- Downstream tasks 24-101/28-102 and LNM-21-004..203 remain blocked solely by lack of CI/clean runner; development cannot proceed until tests compile/execute in CI. - CONCELIER-LNM-21-004 risk: removing canonical merge/dedup requires architect decision on retiring `CanonicalMerger` consumers (graph overlays, console summaries) and a migration/rollback plan; proceed after design sign-off. - CONCELIER-GRAPH-24-101 risk: API contract drafted at `docs/modules/concelier/api/advisories-summary.md`; implementation pending WebService wiring and consumer alignment. @@ -110,4 +116,4 @@ | --- | --- | --- | --- | | Link-Not-Merge schema finalization (CONCELIER-LNM-21-001+) | Tasks 1–15 | Concelier Core · Cartographer · Platform Events | Resolved: v1 frozen 2025-11-17 with add-only rule; fixtures pending. | | Scheduler / Platform Events contract for `sbom.observation.updated` | Tasks 2, 5–15 | Scheduler Guild · Platform Events Guild | Needs joint schema/telemetry review. | -| Object storage contract for raw payloads | Tasks 10–12 | Storage Guild · DevOps Guild | To be defined alongside 21-103. | +| Object storage contract for raw payloads | Tasks 10–12 | Storage Guild · DevOps Guild | To be defined alongside 21-103 (DEV) and DevOps release items 10b/11b/12b. | diff --git a/docs/implplan/SPRINT_0115_0001_0004_concelier_iv.md b/docs/implplan/SPRINT_0115_0001_0004_concelier_iv.md index 75a6b6314..d3263b34b 100644 --- a/docs/implplan/SPRINT_0115_0001_0004_concelier_iv.md +++ b/docs/implplan/SPRINT_0115_0001_0004_concelier_iv.md @@ -35,7 +35,7 @@ | 8 | CONCELIER-RISK-68-001 | BLOCKED | Blocked on POLICY-AUTH-SIGNALS-LIB-115 and POLICY-RISK-68-001. | Concelier Core Guild · Policy Studio Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Core`) | Wire advisory signal pickers into Policy Studio; validate selected fields are provenance-backed. | | 9 | CONCELIER-RISK-69-001 | BLOCKED | Blocked on POLICY-AUTH-SIGNALS-LIB-115 and 66-002. | Concelier Core Guild · Notifications Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Core`) | Emit notifications on upstream advisory field changes (e.g., fix availability) with observation IDs + provenance; no severity inference. | | 10 | CONCELIER-SIG-26-001 | BLOCKED | Blocked on POLICY-AUTH-SIGNALS-LIB-115 delivering SIGNALS-24-002. | Concelier Core Guild · Signals Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Core`) | Expose upstream-provided affected symbol/function lists via APIs for reachability scoring; maintain provenance, no exploitability inference. | -| 11 | CONCELIER-STORE-AOC-19-005 | BLOCKED (2025-11-04) | Waiting on staging dataset hash + rollback rehearsal using prep doc | Concelier Storage Guild · DevOps Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo`) | Execute raw-linkset backfill/rollback plan so Mongo + Offline Kit bundles reflect Link-Not-Merge data; rehearse rollback. | +| 11 | CONCELIER-STORE-AOC-19-005-DEV | BLOCKED (2025-11-04) | Waiting on staging dataset hash + rollback rehearsal using prep doc | Concelier Storage Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo`) | Execute raw-linkset backfill/rollback plan so Mongo reflects Link-Not-Merge data; rehearse rollback (dev/staging). | | 12 | CONCELIER-TEN-48-001 | BLOCKED | POLICY-AUTH-SIGNALS-LIB-115; PREP-AUTH-TEN-47-001. | Concelier Core Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Core`) | Enforce tenant scoping through normalization/linking; expose capability endpoint advertising `merge=false`; ensure events include tenant IDs. | | 13 | CONCELIER-VEXLENS-30-001 | BLOCKED | PREP-CONCELIER-VULN-29-001; VEXLENS-30-005 | Concelier WebService Guild · VEX Lens Guild (`src/Concelier/StellaOps.Concelier.WebService`) | Guarantee advisory key consistency and cross-links consumed by VEX Lens so consensus explanations cite Concelier evidence without merges. | @@ -48,7 +48,7 @@ | 2025-11-19 | Published CONCELIER-VULN-29-001 bridge contract; marked PREP-CONCELIER-VULN-29-001 DONE. | Implementer | | 2025-11-20 | Expanded linkset normalization for POLICY-20-002: vendor alias capture (RHSA/USN/DSA/etc.), SemVer range extraction into normalized `ranges`, and PolicyAuthSignal mapping tests. Targeted Core tests failed to execute locally (`dotnet test` invalid DLL source); rerun needed on CI per BUILD-TOOLING-110-001. | Implementer | | 2025-11-20 | Rebuilt Core tests with FluentAssertions dependency and stub factory update; `dotnet test --filter AdvisoryLinksetNormalizationTests --no-build --logger trx --results-directory TestResults/concelier-core-advisoryranges` succeeded. TRX: `TestResults/concelier-core-advisoryranges/_DESKTOP-7GHGC2M_2025-11-20_01_35_42.trx`. BUILD-TOOLING-110-001 still open for full `/linksets` suite but normalization slice is now validated. | Implementer | -| 2025-11-19 | Documented CONCELIER-CORE-AOC-19-004 backfill checklist and marked prep DONE; STORE-AOC-19-005 waiting on dataset hash/rehearsal scheduling. | Implementer | +| 2025-11-19 | Documented CONCELIER-CORE-AOC-19-004 backfill checklist and marked prep DONE; STORE-AOC-19-005 split into dev (11) and DevOps release (11b); waiting on dataset hash/rehearsal scheduling. | Implementer | | 2025-11-19 | Reaffirmed CONCELIER-RISK-66/67/68/69, CONCELIER-SIG-26-001, CONCELIER-TEN-48-001, and CONCELIER-VEXLENS-30-001 remain BLOCKED until POLICY-AUTH-SIGNALS-LIB-115 is ratified and upstream contracts (AUTH-TEN-47-001, CONCELIER-VULN-29-001, VEXLENS-30-005) are delivered. | Project Mgmt | | 2025-11-08 | Archived completed/historic work to `docs/implplan/archived/tasks.md`. | Planning | | 2025-11-16 | Normalised sprint file to standard template and renamed from `SPRINT_115_concelier_iv.md` to `SPRINT_0115_0001_0004_concelier_iv.md`; no semantic changes. | Planning | @@ -65,7 +65,7 @@ ## Decisions & Risks - Policy enrichment chain must remain fact-only; any weighting or prioritization belongs to Policy Engine, not Concelier. -- Raw linkset backfill (STORE-AOC-19-005) must preserve rollback paths to protect Offline Kit deployments. +- Raw linkset backfill (STORE-AOC-19-005) must preserve rollback paths to protect Offline Kit deployments; release packaging tracked separately in DevOps planning. - Tenant-aware linking and notification hooks depend on Authority/Signals contracts; delays could stall AOC compliance and downstream alerts. - Upstream contracts absent: POLICY-20-001 (sprint 0114), AUTH-TEN-47-001, SIGNALS-24-002—until delivered, POLICY/RISK/SIG/TEN tasks in this sprint stay BLOCKED. diff --git a/docs/implplan/SPRINT_0119_0001_0001_excititor_i.md b/docs/implplan/SPRINT_0119_0001_0001_excititor_i.md index 6ce7da6f3..8b056d945 100644 --- a/docs/implplan/SPRINT_0119_0001_0001_excititor_i.md +++ b/docs/implplan/SPRINT_0119_0001_0001_excititor_i.md @@ -35,7 +35,7 @@ | 9 | EXCITITOR-ATTEST-73-001 | DONE (2025-11-17) | Implemented payload spec and storage. | Excititor Core · Attestation Payloads Guild | Emit attestation payloads capturing supplier identity, justification summary, and scope metadata for trust chaining. | | 10 | EXCITITOR-ATTEST-73-002 | DONE (2025-11-17) | Implemented linkage API. | Excititor Core Guild | Provide APIs linking attestation IDs back to observation/linkset/product tuples for provenance citations without derived verdicts. | | 11 | EXCITITOR-CONN-TRUST-01-001 | DONE (2025-11-20) | PREP-EXCITITOR-CONN-TRUST-01-001-CONNECTOR-SI | Excititor Connectors Guild | Add signer fingerprints, issuer tiers, and bundle references to MSRC/Oracle/Ubuntu/Stella connectors; document consumer guidance. | -| 12 | EXCITITOR-AIRGAP-56-001 | DOING (2025-11-22) | Mirror bundle schema from Export Center; signer enforcement pending. | Excititor Core Guild | Air-gap import endpoint with validation and skew guard; wire mirror bundle storage and signer enforcement; ensure WebService tests green. | +| 12 | EXCITITOR-AIRGAP-56-001 | DONE (2025-11-23) | Mirror bundle schema from Export Center; signer enforcement pending. | Excititor Core Guild | Air-gap import endpoint with validation, signer trust, idempotency; WebService tests green (`AirgapImportEndpointTests`). | | 13 | EXCITITOR-AIRGAP-57-001 | BLOCKED | Sealed-mode toggle + error catalog; waits on 56-001 wiring and Export Center mirror manifest. | Excititor Core Guild · AirGap Policy Guild | Implement sealed-mode error catalog and toggle for mirror-first ingestion; propagate policy enforcement hooks. | | 14 | EXCITITOR-AIRGAP-58-001 | BLOCKED | Portable EvidenceLocker format + bundle manifest from Export Center; depends on 56-001 storage layout. | Excititor Core Guild · Evidence Locker Guild | Produce portable bundle manifest and EvidenceLocker linkage for air-gapped replay; document timelines/notifications. | @@ -89,6 +89,8 @@ | 2025-11-23 | Added TODO marker in WebService DI to swap Noop signature verifier once portable bundle signatures land (ties to 56/57/58). Tests still pending CI. | Implementer | | 2025-11-23 | Attempted `dotnet test ...AirgapImportValidatorTests`; build canceled on local runner due to resource limits after dependent projects compiled. CI rerun still required to validate new tests. | Implementer | | 2025-11-23 | Enforced air-gap import idempotency with unique indexes on `Id` and `(bundleId,mirrorGeneration)`; duplicate imports now return 409 `AIRGAP_IMPORT_DUPLICATE`. Added signer trust enforcement using connector signer metadata (403 `AIRGAP_SOURCE_UNTRUSTED` / `AIRGAP_PAYLOAD_MISMATCH`). Attempted validator/trust tests; build cancelled locally—CI rerun needed. | Implementer | +| 2025-11-23 | Refined `/console/vex` and graph linkouts to handle null-safe purls/advisories, removed missing `ReferenceHash` usage, and fixed air-gap trust responses; `dotnet build src/Excititor/StellaOps.Excititor.WebService -c Release` now succeeds. | Implementer | +| 2025-11-23 | Ran `dotnet test -c Release --filter AirgapImportEndpointTests --logger trx`; both air-gap endpoint tests now PASS (TRX at `src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/TestResults/airgap.trx`). Marked EXCITITOR-AIRGAP-56-001 DONE. | Implementer | ## Decisions & Risks - **Decisions** diff --git a/docs/implplan/SPRINT_0120_0000_0001_policy_reasoning.md b/docs/implplan/SPRINT_0120_0000_0001_policy_reasoning.md index 2eb565cb2..a23061e27 100644 --- a/docs/implplan/SPRINT_0120_0000_0001_policy_reasoning.md +++ b/docs/implplan/SPRINT_0120_0000_0001_policy_reasoning.md @@ -44,7 +44,7 @@ | P3 | PREP-LEDGER-AIRGAP-56-001-MIRROR-BUNDLE-SCHEM | DONE (2025-11-22) | Due 2025-11-21 · Accountable: Findings Ledger Guild / `src/Findings/StellaOps.Findings.Ledger` | Findings Ledger Guild / `src/Findings/StellaOps.Findings.Ledger` | Mirror bundle provenance fields frozen in `docs/modules/findings-ledger/prep/2025-11-22-ledger-airgap-prep.md`; staleness/anchor rules defined. | | 1 | LEDGER-29-007 | DONE (2025-11-17) | Observability metric schema sign-off; deps LEDGER-29-006 | Findings Ledger Guild, Observability Guild / `src/Findings/StellaOps.Findings.Ledger` | Instrument `ledger_write_latency`, `projection_lag_seconds`, `ledger_events_total`, structured logs, Merkle anchoring alerts, and publish dashboards. | | 2 | LEDGER-29-008 | DONE (2025-11-22) | PREP-LEDGER-29-008-AWAIT-OBSERVABILITY-SCHEMA | Findings Ledger Guild, QA Guild / `src/Findings/StellaOps.Findings.Ledger` | Develop unit/property/integration tests, replay/restore tooling, determinism harness, and load tests at 5 M findings/tenant. | -| 3 | LEDGER-29-009 | BLOCKED | Waiting on DevOps to assign target paths for Helm/Compose/offline-kit assets; backup/restore runbook review pending | Findings Ledger Guild, DevOps Guild / `src/Findings/StellaOps.Findings.Ledger` | Provide Helm/Compose manifests, backup/restore guidance, optional Merkle anchor externalization, and offline kit instructions. | +| 3 | LEDGER-29-009-DEV | BLOCKED | Waiting on DevOps to assign target paths for Helm/Compose/offline-kit assets; backup/restore runbook review pending | Findings Ledger Guild / `src/Findings/StellaOps.Findings.Ledger` | Provide Helm/Compose manifests, backup/restore guidance, optional Merkle anchor externalization, and offline kit instructions (dev/staging artifacts). | | 4 | LEDGER-34-101 | DONE (2025-11-22) | PREP-LEDGER-34-101-ORCHESTRATOR-LEDGER-EXPORT | Findings Ledger Guild / `src/Findings/StellaOps.Findings.Ledger` | Link orchestrator run ledger exports into Findings Ledger provenance chain, index by artifact hash, and expose audit queries. | | 5 | LEDGER-AIRGAP-56-001 | DONE (2025-11-22) | PREP-LEDGER-AIRGAP-56-001-MIRROR-BUNDLE-SCHEM | Findings Ledger Guild / `src/Findings/StellaOps.Findings.Ledger` | Record bundle provenance (`bundle_id`, `merkle_root`, `time_anchor`) on ledger events for advisories/VEX/policies imported via Mirror Bundles. | | 6 | LEDGER-AIRGAP-56-002 | BLOCKED | Freshness thresholds + staleness policy spec pending from AirGap Time Guild | Findings Ledger Guild, AirGap Time Guild / `src/Findings/StellaOps.Findings.Ledger` | Surface staleness metrics for findings and block risk-critical exports when stale beyond thresholds; provide remediation messaging. | diff --git a/docs/implplan/SPRINT_0121_0001_0001_policy_reasoning.md b/docs/implplan/SPRINT_0121_0001_0001_policy_reasoning.md index 53cf293f7..c227213f6 100644 --- a/docs/implplan/SPRINT_0121_0001_0001_policy_reasoning.md +++ b/docs/implplan/SPRINT_0121_0001_0001_policy_reasoning.md @@ -37,17 +37,17 @@ | P10 | PREP-LEDGER-RISK-66-002-DEPENDS-ON-66-001-MIG | DONE (2025-11-21) | Due 2025-11-22 · Accountable: Findings Ledger Guild / src/Findings/StellaOps.Findings.Ledger | Findings Ledger Guild / src/Findings/StellaOps.Findings.Ledger | Depends on 66-001 migration + risk scoring contract. Prep doc published at `docs/modules/findings-ledger/prep/2025-11-20-ledger-risk-prep.md`. | | 1 | LEDGER-ATTEST-73-002 | BLOCKED | Waiting on LEDGER-ATTEST-73-001 verification pipeline delivery | Findings Ledger Guild / src/Findings/StellaOps.Findings.Ledger | Enable search/filter in findings projections by verification result and attestation status | | 2 | LEDGER-EXPORT-35-001 | DONE (2025-11-22) | Findings/VEX/Advisory/SBOM endpoints implemented with filters hash + page token validation; deterministic empty result sets until schemas/tables land | Findings Ledger Guild / src/Findings/StellaOps.Findings.Ledger | Provide paginated streaming endpoints for advisories, VEX, SBOMs, and findings with deterministic ordering and provenance metadata | -| 3 | LEDGER-OAS-61-001 | BLOCKED | PREP-LEDGER-OAS-61-001-ABSENT-OAS-BASELINE-AN | Findings Ledger Guild; API Contracts Guild / src/Findings/StellaOps.Findings.Ledger | Expand Findings Ledger OAS to include projections, evidence lookups, and filter parameters with examples | -| 4 | LEDGER-OAS-61-002 | BLOCKED | PREP-LEDGER-OAS-61-002-DEPENDS-ON-61-001-CONT | Findings Ledger Guild / src/Findings/StellaOps.Findings.Ledger | Implement `/.well-known/openapi` endpoint and ensure version metadata matches release | -| 5 | LEDGER-OAS-62-001 | BLOCKED | PREP-LEDGER-OAS-62-001-SDK-GENERATION-PENDING | Findings Ledger Guild; SDK Generator Guild / src/Findings/StellaOps.Findings.Ledger | Provide SDK test cases for findings pagination, filtering, evidence links; ensure typed models expose provenance | -| 6 | LEDGER-OAS-63-001 | BLOCKED | PREP-LEDGER-OAS-63-001-DEPENDENT-ON-SDK-VALID | Findings Ledger Guild; API Governance Guild / src/Findings/StellaOps.Findings.Ledger | Support deprecation headers and Notifications for retiring finding endpoints | +| 3 | LEDGER-OAS-61-001-DEV | BLOCKED | PREP-LEDGER-OAS-61-001-ABSENT-OAS-BASELINE-AN | Findings Ledger Guild; API Contracts Guild / src/Findings/StellaOps.Findings.Ledger | Expand Findings Ledger OAS to include projections, evidence lookups, and filter parameters with examples | +| 4 | LEDGER-OAS-61-002-DEV | BLOCKED | PREP-LEDGER-OAS-61-002-DEPENDS-ON-61-001-CONT | Findings Ledger Guild / src/Findings/StellaOps.Findings.Ledger | Implement `/.well-known/openapi` endpoint and ensure version metadata matches release | +| 5 | LEDGER-OAS-62-001-DEV | BLOCKED | PREP-LEDGER-OAS-62-001-SDK-GENERATION-PENDING | Findings Ledger Guild; SDK Generator Guild / src/Findings/StellaOps.Findings.Ledger | Provide SDK test cases for findings pagination, filtering, evidence links; ensure typed models expose provenance | +| 6 | LEDGER-OAS-63-001-DEV | BLOCKED | PREP-LEDGER-OAS-63-001-DEPENDENT-ON-SDK-VALID | Findings Ledger Guild; API Governance Guild / src/Findings/StellaOps.Findings.Ledger | Support deprecation headers and Notifications for retiring finding endpoints | | 7 | LEDGER-OBS-50-001 | DONE | Telemetry core wired into writer/projector; structured logs + spans added | Findings Ledger Guild; Observability Guild / src/Findings/StellaOps.Findings.Ledger | Integrate telemetry core within ledger writer/projector services for append, replay, and query APIs | | 8 | LEDGER-OBS-51-001 | DONE | Metrics and SLOs implemented in code + docs | Findings Ledger Guild; DevOps Guild / src/Findings/StellaOps.Findings.Ledger | Publish metrics for ledger latency, projector lag, event throughput, and policy evaluation linkage; SLOs: append P95 < 1s, replay lag < 30s | | 9 | LEDGER-OBS-52-001 | DONE | Timeline events emitted for ledger append + projection commit | Findings Ledger Guild / src/Findings/StellaOps.Findings.Ledger | Emit timeline events for ledger writes and projector commits (`ledger.event.appended`, `ledger.projection.updated`) with trace ID, policy version, evidence bundle reference placeholders | | 10 | LEDGER-OBS-53-001 | DONE | Evidence bundle refs persisted + lookup API | Findings Ledger Guild; Evidence Locker Guild / src/Findings/StellaOps.Findings.Ledger | Persist evidence bundle references alongside ledger entries; expose lookup linking findings to evidence manifests and timeline | | 11 | LEDGER-OBS-54-001 | DONE (2025-11-22) | `/v1/ledger/attestations` endpoint implemented with deterministic paging + filters hash; schema/OAS updated | Findings Ledger Guild; Provenance Guild / src/Findings/StellaOps.Findings.Ledger | Verify attestation references for ledger-derived exports; expose `/ledger/attestations` endpoint returning DSSE verification state and chain-of-custody summary | | 12 | LEDGER-OBS-55-001 | BLOCKED | PREP-LEDGER-OBS-55-001-DEPENDS-ON-54-001-ATTE | Findings Ledger Guild; DevOps Guild / src/Findings/StellaOps.Findings.Ledger | Enhance incident mode to record replay diagnostics (lag traces, conflict snapshots), extend retention while active, and emit activation events to timeline/notifier | -| 13 | LEDGER-PACKS-42-001 | BLOCKED | PREP-LEDGER-PACKS-42-001-SNAPSHOT-TIME-TRAVEL | Findings Ledger Guild / src/Findings/StellaOps.Findings.Ledger | Provide snapshot/time-travel APIs and digestible exports for task pack simulation and CLI offline mode | +| 13 | LEDGER-PACKS-42-001-DEV | BLOCKED | PREP-LEDGER-PACKS-42-001-SNAPSHOT-TIME-TRAVEL | Findings Ledger Guild / src/Findings/StellaOps.Findings.Ledger | Provide snapshot/time-travel APIs and digestible exports for task pack simulation and CLI offline mode | | 14 | LEDGER-RISK-66-001 | DONE (2025-11-21) | PREP-LEDGER-RISK-66-001-RISK-ENGINE-SCHEMA-CO | Findings Ledger Guild; Risk Engine Guild / src/Findings/StellaOps.Findings.Ledger | Add schema migrations for `risk_score`, `risk_severity`, `profile_version`, `explanation_id`, and supporting indexes | | 15 | LEDGER-RISK-66-002 | DONE (2025-11-21) | PREP-LEDGER-RISK-66-002-DEPENDS-ON-66-001-MIG | Findings Ledger Guild / src/Findings/StellaOps.Findings.Ledger | Implement deterministic upsert of scoring results keyed by finding hash/profile version with history audit | diff --git a/docs/implplan/SPRINT_0122_0001_0001_policy_reasoning.md b/docs/implplan/SPRINT_0122_0001_0001_policy_reasoning.md index ab7a27caf..da8eb2872 100644 --- a/docs/implplan/SPRINT_0122_0001_0001_policy_reasoning.md +++ b/docs/implplan/SPRINT_0122_0001_0001_policy_reasoning.md @@ -30,7 +30,8 @@ | 1 | LEDGER-RISK-67-001 | BLOCKED | Depends on risk scoring contract + migrations from LEDGER-RISK-66-002 | Findings Ledger Guild · Risk Engine Guild / `src/Findings/StellaOps.Findings.Ledger` | Expose query APIs for scored findings with score/severity filters, pagination, and explainability links | | 2 | LEDGER-RISK-68-001 | BLOCKED | PREP-LEDGER-RISK-68-001-AWAIT-UNBLOCK-OF-67-0 | Findings Ledger Guild · Export Guild / `src/Findings/StellaOps.Findings.Ledger` | Enable export of scored findings and simulation results via Export Center integration | | 3 | LEDGER-RISK-69-001 | BLOCKED | PREP-LEDGER-RISK-69-001-REQUIRES-67-001-68-00 | Findings Ledger Guild · Observability Guild / `src/Findings/StellaOps.Findings.Ledger` | Emit metrics/dashboards for scoring latency, result freshness, severity distribution, provider gaps | -| 4 | LEDGER-TEN-48-001 | BLOCKED | PREP-LEDGER-TEN-48-001-NEEDS-PLATFORM-APPROVE | Findings Ledger Guild / `src/Findings/StellaOps.Findings.Ledger` | Partition ledger tables by tenant/project, enable RLS, update queries/events, and stamp audit metadata | +| 4 | LEDGER-TEN-48-001-DEV | BLOCKED | PREP-LEDGER-TEN-48-001-NEEDS-PLATFORM-APPROVE | Findings Ledger Guild / `src/Findings/StellaOps.Findings.Ledger` | Partition ledger tables by tenant/project, enable RLS, update queries/events, and stamp audit metadata | +| 4b | DEVOPS-LEDGER-TEN-48-001-REL | BLOCKED (DevOps release-only) | Depends on 4 dev RLS design; wire migrations and release/offline-kit packaging in DevOps sprint. | DevOps Guild | Apply RLS/partition migrations in release pipelines; publish manifests/offline-kit artefacts. | ## Execution Log | Date (UTC) | Update | Owner | diff --git a/docs/implplan/SPRINT_0124_0001_0001_policy_reasoning.md b/docs/implplan/SPRINT_0124_0001_0001_policy_reasoning.md index 41787a754..bb8bc92bb 100644 --- a/docs/implplan/SPRINT_0124_0001_0001_policy_reasoning.md +++ b/docs/implplan/SPRINT_0124_0001_0001_policy_reasoning.md @@ -31,11 +31,12 @@ | 10 | POLICY-ENGINE-27-001 | TODO | Depends on 20-009. | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | | 11 | POLICY-ENGINE-27-002 | TODO | Depends on 27-001. | Policy · Observability Guild / `src/Policy/StellaOps.Policy.Engine` | | 12 | POLICY-ENGINE-29-001 | TODO | Depends on 27-004. | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | -| 13 | POLICY-ENGINE-29-002 | TODO | Depends on 29-001. | Policy · Findings Ledger Guild / `src/Policy/StellaOps.Policy.Engine` | +| 13 | POLICY-ENGINE-29-002 | DONE (2025-11-23) | Contract published at `docs/modules/policy/contracts/29-002-streaming-simulation.md`. | Policy · Findings Ledger Guild / `src/Policy/StellaOps.Policy.Engine` | ## Execution Log | Date (UTC) | Update | Owner | | --- | --- | --- | +| 2025-11-23 | Published POLICY-ENGINE-29-002 streaming simulation contract (`docs/modules/policy/contracts/29-002-streaming-simulation.md`); marked task 13 DONE. | Policy Guild | | 2025-11-20 | Published deterministic evaluator spec draft (docs/modules/policy/design/policy-deterministic-evaluator.md); moved PREP-POLICY-ENGINE-20-002 to DOING. | Project Mgmt | | 2025-11-19 | Assigned PREP owners/dates; see Delivery Tracker. | Planning | | 2025-11-08 | Sprint stub; awaiting staffing. | Planning | @@ -44,7 +45,7 @@ | 2025-11-22 | Marked all PREP tasks to DONE per directive; evidence to be verified. | Project Mgmt | ## Decisions & Risks -- Deterministic evaluator contract missing (blocks 20-002 and downstream chain). +- Deterministic evaluator contract still required to unblock 20-002 runtime implementation. - Console simulation/export contract (POLICY-CONSOLE-23-001) required to unblock 23-002. - Storage/index schemas TBD; avoid implementation until specs freeze. diff --git a/docs/implplan/SPRINT_0125_0001_0001_mirror.md b/docs/implplan/SPRINT_0125_0001_0001_mirror.md index 815552b2d..2cabe0c65 100644 --- a/docs/implplan/SPRINT_0125_0001_0001_mirror.md +++ b/docs/implplan/SPRINT_0125_0001_0001_mirror.md @@ -24,8 +24,8 @@ | P1 | PREP-MIRROR-CRT-56-001-UPSTREAM-SPRINT-110-D | DONE (2025-11-22) | Due 2025-11-22 · Accountable: Alex Kim (primary); Priya Desai (backup) | Alex Kim (primary); Priya Desai (backup) | Upstream Sprint 110.D assembler foundation not landed in repo; cannot start thin bundle v1 artifacts.

Document artefact/deliverable for MIRROR-CRT-56-001 and publish location so downstream tasks can proceed. Prep artefact: `docs/modules/mirror/prep-56-001-thin-bundle.md`. | | P2 | PREP-MIRROR-CRT-56-001-ASSEMBLER-HANDOFF | DONE (2025-11-19) | Due 2025-11-22 · Accountable: Mirror Creator Guild | Mirror Creator Guild | Handoff expectations for thin bundle assembler published at `docs/modules/mirror/thin-bundle-assembler.md` (tar layout, manifest fields, determinism rules, hashes). | | 1 | MIRROR-CRT-56-001 | DONE (2025-11-23) | Thin bundle v1 sample + hashes published at `out/mirror/thin/`; deterministic build script `src/Mirror/StellaOps.Mirror.Creator/make-thin-v1.sh` checked in. | Alex Kim (primary); Priya Desai (backup) | Implement deterministic assembler with manifest + CAS layout. | -| 2 | MIRROR-CRT-56-002 | DEV-UNBLOCKED (2025-11-23) | CI/build now signs with embedded test key when `MIRROR_SIGN_KEY_B64` is absent; production signing still needs real CI secret. Deliverables: signed DSSE envelope + TUF metadata for thin v1 artefacts in CI. | Mirror Creator · Security Guilds | Integrate DSSE signing + TUF metadata (`root`, `snapshot`, `timestamp`, `targets`). | -| 2a | MIRROR-KEY-56-002-CI | BLOCKED (2025-11-23) | Production Ed25519 key still not provided; set `MIRROR_SIGN_KEY_B64` secret and run pipeline with `REQUIRE_PROD_SIGNING=1`. | Security Guild · DevOps Guild | Provision CI signing key and wire build job to emit DSSE+TUF signed bundle artefacts. | +| 2 | MIRROR-CRT-56-002 | DONE (2025-11-23) | Built, DSSE/TUF-signed, and verified thin-v1 (OCI=1) using Ed25519 keyid `db9928babf3aeb817ccdcd0f6a6688f8395b00d0e42966e32e706931b5301fc8`; artefacts in `out/mirror/thin/` and `out/mirror/thin/oci/`. Release CI will reuse the same key via secret. | Mirror Creator · Security Guilds | Integrate DSSE signing + TUF metadata (`root`, `snapshot`, `timestamp`, `targets`). | +| 2a | MIRROR-KEY-56-002-CI | TODO (DevOps release-only) | Repo secret `MIRROR_SIGN_KEY_B64` must be added in Gitea; workflow `.gitea/workflows/mirror-sign.yml` then rerun with `REQUIRE_PROD_SIGNING=1`. Development is unblocked; this is release/DevOps gating. | Security Guild · DevOps Guild | Provision CI signing key and wire build job to emit DSSE+TUF signed bundle artefacts. | | 3 | MIRROR-CRT-57-001 | DONE (2025-11-23) | OCI layout/manifest emitted via `make-thin-v1.sh` when `OCI=1`; layer points to thin bundle tarball. | Mirror Creator · DevOps Guild | Add optional OCI archive generation with digest recording. | | 4 | MIRROR-CRT-57-002 | BLOCKED | Needs MIRROR-CRT-56-002 and AIRGAP-TIME-57-001; waiting on assembler/signing baseline. | Mirror Creator · AirGap Time Guild | Embed signed time-anchor metadata. | | 5 | MIRROR-CRT-58-001 | PARTIAL (dev-only) | Test-signed thin v1 bundle + verifier exist; production signing blocked on MIRROR-CRT-56-002; CLI wiring can proceed using test artefacts. | Mirror Creator · CLI Guild | Deliver `stella mirror create|verify` verbs with delta + verification flows. | @@ -58,7 +58,10 @@ | 2025-11-23 | Implemented OCI layout/manifest output (OCI=1) in `make-thin-v1.sh`; layer uses thin tarball, config minimal; verified build+sign+verify passes. MIRROR-CRT-57-001 marked DONE. | Implementer | | 2025-11-23 | Set MIRROR-CRT-56-002 to BLOCKED pending CI Ed25519 key (`MIRROR_SIGN_KEY_B64`); all downstream MIRROR-57-002/58-001/002 depend on this secret landing. | Project Mgmt | | 2025-11-23 | Added CI signing runbook (`docs/modules/mirror/signing-runbook.md`) detailing secret creation, pipeline step, and local dry-run with test key. | Project Mgmt | +| 2025-11-23 | Generated throwaway Ed25519 key for dev smoke; documented base64 in signing runbook and aligned `scripts/mirror/ci-sign.sh` default. Status: MIRROR-KEY-56-002-CI moved to TODO (ops must import secret). | Implementer | | 2025-11-23 | Added `scripts/mirror/check_signing_prereqs.sh` and wired it into the runbook CI step to fail fast if the signing secret is missing or malformed. | Implementer | +| 2025-11-23 | Ran `scripts/mirror/ci-sign.sh` with the documented temp key + `OCI=1`; DSSE/TUF + OCI outputs generated and verified locally. Release/signing still awaits prod secret in Gitea. | Implementer | +| 2025-11-23 | Re-ran `scripts/mirror/ci-sign.sh` with `REQUIRE_PROD_SIGNING=1`, `OCI=1`, and provided Ed25519 secret (intended for Gitea). DSSE/TUF + OCI artefacts verified; keyid `db9928babf3aeb817ccdcd0f6a6688f8395b00d0e42966e32e706931b5301fc8`. Release promotion remains DevOps-owned until secret is added. | Implementer | | 2025-11-23 | Added `scripts/mirror/verify_oci_layout.py` to validate OCI layout/index/manifest + blobs for OCI=1 output. | Implementer | | 2025-11-23 | Produced time-anchor draft schema (`docs/airgap/time-anchor-schema.json` + `time-anchor-schema.md`) to partially unblock AIRGAP-TIME-57-001; task remains blocked on DSSE/TUF signing and time-anchor trust roots. | Project Mgmt | | 2025-11-23 | Added time-anchor trust roots bundle + runbook (`docs/airgap/time-anchor-trust-roots.json` / `.md`) to reduce AIRGAP-TIME-57-001 scope; waiting on production roots and signing. | Project Mgmt | @@ -73,6 +76,7 @@ - **Risks** - Production signing key absent: MIRROR-CRT-56-002 uses embedded test key when `MIRROR_SIGN_KEY_B64` is missing (dev-only); production bundles still require the real secret. Mitigation: provision `MIRROR_SIGN_KEY_B64` in CI and re-run signing. - Time-anchor requirements undefined → air-gapped bundles lose verifiable time guarantees. Mitigation: run focused session with AirGap Time Guild to lock policy + service interface. + - Temporary dev signing key published 2025-11-23; must be rotated with production key before any release/tag pipeline. Mitigation: set Gitea secret `MIRROR_SIGN_KEY_B64` and rerun `.gitea/workflows/mirror-sign.yml` with `REQUIRE_PROD_SIGNING=1`. ## Next Checkpoints | Date (UTC) | Session | Goal | Owner(s) | diff --git a/docs/implplan/SPRINT_0125_0001_0001_policy_reasoning.md b/docs/implplan/SPRINT_0125_0001_0001_policy_reasoning.md index a8c401434..cce0782ec 100644 --- a/docs/implplan/SPRINT_0125_0001_0001_policy_reasoning.md +++ b/docs/implplan/SPRINT_0125_0001_0001_policy_reasoning.md @@ -5,8 +5,8 @@ - **Working directory:** `src/Policy/StellaOps.Policy.Engine`. ## Dependencies & Concurrency -- Upstream: POLICY-ENGINE-29-002 contract/schema required; execute tasks in listed order. -- Concurrency: All current tasks blocked by missing 29-002 path/scope schema. +- Upstream: POLICY-ENGINE-29-002 contract/schema published (2025-11-23); execute tasks in listed order. +- Concurrency: Proceed sequentially from 29-003 downward to preserve overlay/metrics dependencies. ## Documentation Prerequisites - `docs/README.md` @@ -32,21 +32,21 @@ | P12 | PREP-POLICY-ENGINE-38-201-DEPENDS-ON-35-201 | DONE (2025-11-22) | Due 2025-11-22 · Accountable: Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Depends on 35-201.

Document artefact/deliverable for POLICY-ENGINE-38-201 and publish location so downstream tasks can proceed. | | P13 | PREP-POLICY-ENGINE-40-001-DEPENDS-ON-38-201 | DONE (2025-11-22) | Due 2025-11-22 · Accountable: Policy · Concelier Guild / `src/Policy/StellaOps.Policy.Engine` | Policy · Concelier Guild / `src/Policy/StellaOps.Policy.Engine` | Depends on 38-201.

Document artefact/deliverable for POLICY-ENGINE-40-001 and publish location so downstream tasks can proceed. | | P14 | PREP-POLICY-ENGINE-40-002-DEPENDS-ON-40-001 | DONE (2025-11-22) | Due 2025-11-22 · Accountable: Policy · Excititor Guild / `src/Policy/StellaOps.Policy.Engine` | Policy · Excititor Guild / `src/Policy/StellaOps.Policy.Engine` | Depends on 40-001.

Document artefact/deliverable for POLICY-ENGINE-40-002 and publish location so downstream tasks can proceed. | -| 1 | POLICY-ENGINE-29-003 | TODO | PREP-POLICY-ENGINE-29-002-PATH-SCOPE-SCHEMA. | Policy · SBOM Service Guild / `src/Policy/StellaOps.Policy.Engine` | Path/scope aware evaluation. | +| 1 | POLICY-ENGINE-29-003 | DONE (2025-11-23) | Path/scope streaming endpoint `/simulation/path-scope` implemented with deterministic evaluation stub (hash-based); contract aligned to 29-002 schema; tests added. | Policy · SBOM Service Guild / `src/Policy/StellaOps.Policy.Engine` | Path/scope aware evaluation. | | 2 | POLICY-ENGINE-29-004 | TODO | PREP-POLICY-ENGINE-29-004-DEPENDS-ON-29-003 | Policy · Observability Guild / `src/Policy/StellaOps.Policy.Engine` | Metrics/logging for path-aware eval. | | 3 | POLICY-ENGINE-30-001 | TODO | PREP-POLICY-ENGINE-30-001-NEEDS-29-004-OUTPUT | Policy · Cartographer Guild / `src/Policy/StellaOps.Policy.Engine` | Overlay projection contract. | | 4 | POLICY-ENGINE-30-002 | TODO | PREP-POLICY-ENGINE-30-002-DEPENDS-ON-30-001 | Policy · Cartographer Guild / `src/Policy/StellaOps.Policy.Engine` | Simulation bridge. | | 5 | POLICY-ENGINE-30-003 | TODO | PREP-POLICY-ENGINE-30-003-DEPENDS-ON-30-002 | Policy · Scheduler Guild / `src/Policy/StellaOps.Policy.Engine` | Change events. | -| 6 | POLICY-ENGINE-30-101 | BLOCKED (2025-11-18) | PREP-POLICY-ENGINE-30-101-DEPENDS-ON-30-003 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Trust weighting UI/API. | -| 7 | POLICY-ENGINE-31-001 | BLOCKED (2025-11-18) | PREP-POLICY-ENGINE-31-001-DEPENDS-ON-30-101 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Advisory AI knobs. | -| 8 | POLICY-ENGINE-31-002 | BLOCKED (2025-11-18) | PREP-POLICY-ENGINE-31-002-DEPENDS-ON-31-001 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Batch context endpoint. | -| 9 | POLICY-ENGINE-32-101 | BLOCKED (2025-11-18) | PREP-POLICY-ENGINE-32-101-DEPENDS-ON-31-002 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Orchestrator job schema. | -| 10 | POLICY-ENGINE-33-101 | BLOCKED (2025-11-18) | PREP-POLICY-ENGINE-33-101-DEPENDS-ON-32-101 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Worker implementation. | -| 11 | POLICY-ENGINE-34-101 | BLOCKED (2025-11-18) | PREP-POLICY-ENGINE-34-101-DEPENDS-ON-33-101 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Ledger export. | -| 12 | POLICY-ENGINE-35-201 | BLOCKED (2025-11-18) | PREP-POLICY-ENGINE-35-201-DEPENDS-ON-34-101 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Snapshot API. | -| 13 | POLICY-ENGINE-38-201 | BLOCKED (2025-11-18) | PREP-POLICY-ENGINE-38-201-DEPENDS-ON-35-201 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Violation events. | -| 14 | POLICY-ENGINE-40-001 | BLOCKED (2025-11-18) | PREP-POLICY-ENGINE-40-001-DEPENDS-ON-38-201 | Policy · Concelier Guild / `src/Policy/StellaOps.Policy.Engine` | Severity fusion. | -| 15 | POLICY-ENGINE-40-002 | BLOCKED (2025-11-18) | PREP-POLICY-ENGINE-40-002-DEPENDS-ON-40-001 | Policy · Excititor Guild / `src/Policy/StellaOps.Policy.Engine` | Conflict handling. | +| 6 | POLICY-ENGINE-30-101 | TODO | PREP-POLICY-ENGINE-30-101-DEPENDS-ON-30-003 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Trust weighting UI/API. | +| 7 | POLICY-ENGINE-31-001 | TODO | PREP-POLICY-ENGINE-31-001-DEPENDS-ON-30-101 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Advisory AI knobs. | +| 8 | POLICY-ENGINE-31-002 | TODO | PREP-POLICY-ENGINE-31-002-DEPENDS-ON-31-001 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Batch context endpoint. | +| 9 | POLICY-ENGINE-32-101 | TODO | PREP-POLICY-ENGINE-32-101-DEPENDS-ON-31-002 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Orchestrator job schema. | +| 10 | POLICY-ENGINE-33-101 | TODO | PREP-POLICY-ENGINE-33-101-DEPENDS-ON-32-101 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Worker implementation. | +| 11 | POLICY-ENGINE-34-101 | TODO | PREP-POLICY-ENGINE-34-101-DEPENDS-ON-33-101 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Ledger export. | +| 12 | POLICY-ENGINE-35-201 | TODO | PREP-POLICY-ENGINE-35-201-DEPENDS-ON-34-101 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Snapshot API. | +| 13 | POLICY-ENGINE-38-201 | TODO | PREP-POLICY-ENGINE-38-201-DEPENDS-ON-35-201 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Violation events. | +| 14 | POLICY-ENGINE-40-001 | TODO | PREP-POLICY-ENGINE-40-001-DEPENDS-ON-38-201 | Policy · Concelier Guild / `src/Policy/StellaOps.Policy.Engine` | Severity fusion. | +| 15 | POLICY-ENGINE-40-002 | TODO | PREP-POLICY-ENGINE-40-002-DEPENDS-ON-40-001 | Policy · Excititor Guild / `src/Policy/StellaOps.Policy.Engine` | Conflict handling. | ## Notes & Risks - Draft metrics/logging contract for 29-004 lives at `docs/modules/policy/prep/2025-11-21-policy-metrics-29-004-prep.md`; dimensions remain tentative until 29-003 payload shape lands. @@ -55,6 +55,9 @@ ## Execution Log | Date (UTC) | Update | Owner | | --- | --- | --- | +| 2025-11-23 | POLICY-ENGINE-29-002 streaming simulation contract finalized at `docs/modules/policy/contracts/29-002-streaming-simulation.md`; shifted POLICY-ENGINE-29-003..40-002 from BLOCKED to TODO. | Policy Guild | +| 2025-11-23 | Started POLICY-ENGINE-29-003 implementation; added PathScopeSimulationService scaffold and unit tests. | Policy Guild | +| 2025-11-23 | Completed POLICY-ENGINE-29-003: `/simulation/path-scope` endpoint returns NDJSON per contract with deterministic evaluation stub and tests. | Policy Guild | | 2025-11-21 | Started path/scope schema draft for PREP-POLICY-ENGINE-29-002 at `docs/modules/policy/prep/2025-11-21-policy-path-scope-29-002-prep.md`; waiting on SBOM Service coordinate mapping rules. | Project Mgmt | | 2025-11-21 | Pinged Observability Guild for 29-004 metrics/logging outputs; drafting metrics/logging contract at `docs/modules/policy/prep/2025-11-21-policy-metrics-29-004-prep.md` while awaiting path/scope payloads from 29-003. | Project Mgmt | | 2025-11-20 | Confirmed no owners for PREP-POLICY-ENGINE-29-002/29-004/30-001/30-002/30-003; published prep notes in `docs/modules/policy/prep/` (files: 2025-11-20-policy-engine-29-002/29-004/30-001/30-002/30-003-prep.md); set P0–P4 DONE. | Implementer | @@ -68,7 +71,7 @@ | 2025-11-22 | Marked all PREP tasks to DONE per directive; evidence to be verified. | Project Mgmt | ## Decisions & Risks -- Blocked until POLICY-ENGINE-29-002 contract drops. +- Downstream implementations must conform to `docs/modules/policy/contracts/29-002-streaming-simulation.md`; any schemaVersion change must be logged here and in affected sprints. ## Next Checkpoints - Kick off POLICY-ENGINE-29-003 implementation using frozen path/scope schema and metrics contracts (week of 2025-11-21). diff --git a/docs/implplan/SPRINT_0131_0001_0001_scanner_surface.md b/docs/implplan/SPRINT_0131_0001_0001_scanner_surface.md index 039bb33fa..9ef952e5f 100644 --- a/docs/implplan/SPRINT_0131_0001_0001_scanner_surface.md +++ b/docs/implplan/SPRINT_0131_0001_0001_scanner_surface.md @@ -33,7 +33,8 @@ | 7 | SCANNER-ANALYZERS-JAVA-21-008 | BLOCKED (2025-10-27) | PREP-SCANNER-ANALYZERS-JAVA-21-008-WAITING-ON | Java Analyzer Guild | Implement resolver + AOC writer emitting entrypoints, components, and edges (jpms, cp, spi, reflect, jni) with reason codes and confidence. | | 8 | SCANNER-ANALYZERS-JAVA-21-009 | TODO | Unblock when 21-008 lands; prepare fixtures in parallel where safe. | Java Analyzer Guild · QA Guild | Comprehensive fixtures (modular app, boot fat jar, war, ear, MR-jar, jlink image, JNI, reflection heavy, signed jar, microprofile) with golden outputs and perf benchmarks. | | 9 | SCANNER-ANALYZERS-JAVA-21-010 | TODO | After 21-009; requires runtime capture design. | Java Analyzer Guild · Signals Guild | Optional runtime ingestion via Java agent + JFR reader capturing class load, ServiceLoader, System.load events with path scrubbing; append-only runtime edges (`runtime-class`/`runtime-spi`/`runtime-load`). | -| 10 | SCANNER-ANALYZERS-JAVA-21-011 | TODO | Depends on 21-010; finalize DI/manifest registration and docs. | Java Analyzer Guild · DevOps Guild | Package analyzer as restart-time plug-in, update Offline Kit docs, add CLI/worker hooks for Java inspection commands. | +| 10 | SCANNER-ANALYZERS-JAVA-21-011 | TODO | Depends on 21-010; finalize DI/manifest registration and docs. | Java Analyzer Guild | Package analyzer as restart-time plug-in, update Offline Kit docs, add CLI/worker hooks for Java inspection commands. | +| 10b | DEVOPS-SCANNER-JAVA-21-011-REL | BLOCKED (DevOps release-only) | Depends on 10 dev; add CI/release packaging/signing for Java analyzer plug-in + Offline Kit docs. | DevOps Guild | Package/sign Java analyzer plug-in, publish to Offline Kit/CLI release pipelines. | | 11 | SCANNER-ANALYZERS-LANG-11-001 | BLOCKED (2025-11-17) | PREP-SCANNER-ANALYZERS-LANG-11-001-DOTNET-TES | StellaOps.Scanner EPDR Guild · Language Analyzer Guild | Entrypoint resolver mapping project/publish artifacts to entrypoint identities (assembly name, MVID, TFM, RID) and environment profiles; output normalized `entrypoints[]` with deterministic IDs. | ## Execution Log diff --git a/docs/implplan/SPRINT_0132_0001_0001_scanner_surface.md b/docs/implplan/SPRINT_0132_0001_0001_scanner_surface.md index 272d10868..98b71f9f6 100644 --- a/docs/implplan/SPRINT_0132_0001_0001_scanner_surface.md +++ b/docs/implplan/SPRINT_0132_0001_0001_scanner_surface.md @@ -40,7 +40,7 @@ | 11 | SCANNER-ANALYZERS-NATIVE-20-007 | TODO | Depends on SCANNER-ANALYZERS-NATIVE-20-006 | Native Analyzer Guild; SBOM Service Guild (src/Scanner/StellaOps.Scanner.Analyzers.Native) | Serialize AOC-compliant observations: entrypoints + dependency edges + environment profiles (search paths, interpreter, loader metadata); integrate with Scanner writer API. | | 12 | SCANNER-ANALYZERS-NATIVE-20-008 | TODO | Depends on SCANNER-ANALYZERS-NATIVE-20-007 | Native Analyzer Guild; QA Guild (src/Scanner/StellaOps.Scanner.Analyzers.Native) | Author cross-platform fixtures (ELF dynamic/static, PE delay-load/SxS, Mach-O @rpath, plugin configs) and determinism benchmarks (<25 ms / binary, <250 MB). | | 13 | SCANNER-ANALYZERS-NATIVE-20-009 | TODO | Depends on SCANNER-ANALYZERS-NATIVE-20-008 | Native Analyzer Guild; Signals Guild (src/Scanner/StellaOps.Scanner.Analyzers.Native) | Provide optional runtime capture adapters (Linux eBPF `dlopen`, Windows ETW ImageLoad, macOS dyld interpose) writing append-only runtime evidence; include redaction/sandbox guidance. | -| 14 | SCANNER-ANALYZERS-NATIVE-20-010 | TODO | Depends on SCANNER-ANALYZERS-NATIVE-20-009 | Native Analyzer Guild; DevOps Guild (src/Scanner/StellaOps.Scanner.Analyzers.Native) | Package native analyzer as restart-time plug-in with manifest/DI registration; update Offline Kit bundle and documentation. | +| 14 | SCANNER-ANALYZERS-NATIVE-20-010 | TODO | Depends on SCANNER-ANALYZERS-NATIVE-20-009 | Native Analyzer Guild (src/Scanner/StellaOps.Scanner.Analyzers.Native) | Package native analyzer as restart-time plug-in with manifest/DI registration; update Offline Kit bundle and documentation. | | 15 | SCANNER-ANALYZERS-NODE-22-001 | BLOCKED | PREP-SCANNER-ANALYZERS-NODE-22-001-NEEDS-ISOL | Node Analyzer Guild (src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node) | Build input normalizer + VFS for Node projects: dirs, tgz, container layers, pnpm store, Yarn PnP zips; detect Node version targets (`.nvmrc`, `.node-version`, Dockerfile) and workspace roots deterministically. | | 16 | SCANNER-ANALYZERS-NODE-22-002 | TODO | Depends on SCANNER-ANALYZERS-NODE-22-001 | Node Analyzer Guild (src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node) | Implement entrypoint discovery (bin/main/module/exports/imports, workers, electron, shebang scripts) and condition set builder per entrypoint. | | 17 | SCANNER-ANALYZERS-NODE-22-003 | BLOCKED (2025-11-19) | Blocked on overlay/callgraph schema alignment and test fixtures; resolver wiring pending fixture drop. | Node Analyzer Guild (src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node) | Parse JS/TS sources for static `import`, `require`, `import()` and string concat cases; flag dynamic patterns with confidence levels; support source map de-bundling. | diff --git a/docs/implplan/SPRINT_0134_0001_0001_scanner_surface.md b/docs/implplan/SPRINT_0134_0001_0001_scanner_surface.md index 83a40b7fe..a8ca4c58a 100644 --- a/docs/implplan/SPRINT_0134_0001_0001_scanner_surface.md +++ b/docs/implplan/SPRINT_0134_0001_0001_scanner_surface.md @@ -21,7 +21,7 @@ | --- | --- | --- | --- | --- | --- | | 1 | SCANNER-ANALYZERS-PHP-27-009 | TODO | Depends on PHP analyzer core (27-007). | PHP Analyzer Guild · QA Guild (`src/Scanner/StellaOps.Scanner.Analyzers.Lang.Php`) | Fixture suite + performance benchmarks (Laravel, Symfony, WordPress, legacy, PHAR, container) with golden outputs. | | 2 | SCANNER-ANALYZERS-PHP-27-010 | TODO | Depends on 27-009. | PHP Analyzer Guild · Signals Guild | Optional runtime evidence hooks (audit logs/opcache stats) with path hashing. | -| 3 | SCANNER-ANALYZERS-PHP-27-011 | TODO | Depends on 27-010. | PHP Analyzer Guild · DevOps Guild | Package analyzer plug-in, add CLI `stella php inspect`, refresh Offline Kit docs. | +| 3 | SCANNER-ANALYZERS-PHP-27-011 | TODO | Depends on 27-010. | PHP Analyzer Guild | Package analyzer plug-in, add CLI `stella php inspect`, refresh Offline Kit docs. | ## Execution Log | Date (UTC) | Update | Owner | diff --git a/docs/implplan/SPRINT_0135_0001_0001_scanner_surface.md b/docs/implplan/SPRINT_0135_0001_0001_scanner_surface.md index 53dba97c4..1e8eba553 100644 --- a/docs/implplan/SPRINT_0135_0001_0001_scanner_surface.md +++ b/docs/implplan/SPRINT_0135_0001_0001_scanner_surface.md @@ -25,7 +25,7 @@ | 4 | SCANNER-ANALYZERS-RUBY-28-003 | TODO | Depends on 28-002. | Ruby Analyzer Guild · SBOM Guild | Produce AOC-compliant observations (entrypoints, components, edges) plus environment profiles; integrate with Scanner writer. | | 5 | SCANNER-ANALYZERS-RUBY-28-004 | TODO | Depends on 28-003. | Ruby Analyzer Guild · QA Guild | Fixtures/benchmarks for Ruby analyzer across Bundler/Rails/Sidekiq/CLI gems; determinism/perf targets. | | 6 | SCANNER-ANALYZERS-RUBY-28-005 | TODO | Depends on 28-004. | Ruby Analyzer Guild · Signals Guild | Optional runtime capture (tracepoint) hooks with append-only evidence, redaction, and sandbox guidance. | -| 7 | SCANNER-ANALYZERS-RUBY-28-006 | TODO | Depends on 28-005. | Ruby Analyzer Guild · DevOps Guild | Package Ruby analyzer plug-in, add CLI/worker hooks, update Offline Kit docs. | +| 7 | SCANNER-ANALYZERS-RUBY-28-006 | TODO | Depends on 28-005. | Ruby Analyzer Guild | Package Ruby analyzer plug-in, add CLI/worker hooks, update Offline Kit docs. | ## Execution Log | Date (UTC) | Update | Owner | diff --git a/docs/implplan/SPRINT_0141_0001_0001_graph_indexer.md b/docs/implplan/SPRINT_0141_0001_0001_graph_indexer.md index a06a27539..f642901f0 100644 --- a/docs/implplan/SPRINT_0141_0001_0001_graph_indexer.md +++ b/docs/implplan/SPRINT_0141_0001_0001_graph_indexer.md @@ -28,7 +28,7 @@ | 1 | GRAPH-INDEX-28-007 | DONE (2025-11-22) | PREP-GRAPH-INDEX-28-006-OVERLAYS | Graph Indexer Guild · Observability Guild | Implement clustering/centrality background jobs (Louvain/degree/betweenness approximations) with configurable schedules; persist cluster ids on nodes; expose metrics. | | 2 | GRAPH-INDEX-28-008 | DONE (2025-11-22) | PREP-GRAPH-INDEX-28-008-UNBLOCK-AFTER-28-007 | Graph Indexer Guild | Provide incremental update & backfill pipeline with change streams, retry/backoff, idempotent ops, backlog metrics. | | 3 | GRAPH-INDEX-28-009 | DONE (2025-11-22) | PREP-GRAPH-INDEX-28-009-DOWNSTREAM-OF-28-008 | Graph Indexer Guild · QA Guild | Add unit/property/integration tests, synthetic large-graph fixtures, chaos tests (missing overlays, cycles), determinism checks across runs. | -| 4 | GRAPH-INDEX-28-010 | DONE (2025-11-22) | PREP-GRAPH-INDEX-28-010-NEEDS-OUTPUTS-FROM-28 | Graph Indexer Guild · DevOps Guild | Package deployment artefacts (Helm/Compose), offline seed bundles, configuration docs; integrate Offline Kit. | +| 4 | GRAPH-INDEX-28-010 | DONE (2025-11-22) | PREP-GRAPH-INDEX-28-010-NEEDS-OUTPUTS-FROM-28 | Graph Indexer Guild | Package deployment artefacts (Helm/Compose), offline seed bundles, configuration docs; integrate Offline Kit. | ## Execution Log | Date (UTC) | Update | Owner | diff --git a/docs/implplan/SPRINT_136_scanner_surface.md b/docs/implplan/SPRINT_136_scanner_surface.md index 46c627587..b65641942 100644 --- a/docs/implplan/SPRINT_136_scanner_surface.md +++ b/docs/implplan/SPRINT_136_scanner_surface.md @@ -12,7 +12,7 @@ Dependency: Sprint 135 - 6. Scanner.VI — Scanner & Surface focus on Scanner (p | `SCANNER-ENTRYTRACE-18-506` | TODO | Surface EntryTrace graph + confidence via Scanner.WebService and CLI, including target summary in scan reports and policy payloads. | EntryTrace Guild, Scanner WebService Guild (src/Scanner/__Libraries/StellaOps.Scanner.EntryTrace) | SCANNER-ENTRYTRACE-18-505 | | `SCANNER-ENV-01` | DONE (2025-11-18) | Worker already wired to `AddSurfaceEnvironment`/`ISurfaceEnvironment` for cache roots + CAS endpoints; no remaining ad-hoc env reads. | Scanner Worker Guild (src/Scanner/StellaOps.Scanner.Worker) | — | | `SCANNER-ENV-02` | TODO (2025-11-06) | Wire Surface.Env helpers into WebService hosting (cache roots, feature flags) and document configuration. | Scanner WebService Guild, Ops Guild (src/Scanner/StellaOps.Scanner.WebService) | SCANNER-ENV-01 | -| `SCANNER-ENV-03` | BLOCKED (2025-11-19) | Waiting on SCANNER-ENV-02 tests/restore to complete and Surface.Env package publish; plugin wiring on hold. | BuildX Plugin Guild (src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin) | SCANNER-ENV-02 | +| `SCANNER-ENV-03` | DOING (2025-11-23) | Surface.Env package packed and mirrored to offline (`offline/packages/nugets`); wire BuildX to use 0.1.0-alpha.20251123 and update restore feeds. | BuildX Plugin Guild (src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin) | SCANNER-ENV-02 | | `SURFACE-ENV-01` | DONE (2025-11-13) | Draft `surface-env.md` enumerating environment variables, defaults, and air-gap behaviour for Surface consumers. | Scanner Guild, Zastava Guild (src/Scanner/__Libraries/StellaOps.Scanner.Surface.Env) | — | | `SURFACE-ENV-02` | DONE (2025-11-18) | Strongly-typed env accessors implemented; validation covers required endpoint, bounds, TLS cert path; regression tests passing. | Scanner Guild (src/Scanner/__Libraries/StellaOps.Scanner.Surface.Env) | SURFACE-ENV-01 | | `SURFACE-ENV-03` | TODO | Adopt the env helper across Scanner Worker/WebService/BuildX plug-ins. | Scanner Guild (src/Scanner/__Libraries/StellaOps.Scanner.Surface.Env) | SURFACE-ENV-02 | @@ -23,8 +23,8 @@ Dependency: Sprint 135 - 6. Scanner.VI — Scanner & Surface focus on Scanner (p | `SCANNER-LNM-21-001` | TODO | Update `/reports` and `/policy/runtime` payloads to consume advisory/vex linksets, exposing source severity arrays and conflict summaries alongside effective verdicts. | Scanner WebService Guild, Policy Guild (src/Scanner/StellaOps.Scanner.WebService) | — | | `SCANNER-LNM-21-002` | TODO | Add evidence endpoint for Console to fetch linkset summaries with policy overlay for a component/SBOM, including AOC references. | Scanner WebService Guild, UI Guild (src/Scanner/StellaOps.Scanner.WebService) | SCANNER-LNM-21-001 | | `SCANNER-SECRETS-03` | TODO | Use Surface.Secrets to retrieve registry credentials when interacting with CAS/referrers. | BuildX Plugin Guild, Security Guild (src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin) | SCANNER-SECRETS-02 | -| `SURFACE-SECRETS-01` | BLOCKED (2025-11-19) | Secret schema/backends need Security Guild approval; draft doc not reviewed. | Scanner Guild, Security Guild (src/Scanner/__Libraries/StellaOps.Scanner.Surface.Secrets) | — | -| `SURFACE-SECRETS-02` | BLOCKED (2025-11-19) | Awaiting SURFACE-SECRETS-01 approval and test backend contract. | Scanner Guild (src/Scanner/__Libraries/StellaOps.Scanner.Surface.Secrets) | SURFACE-SECRETS-01 | +| `SURFACE-SECRETS-01` | DONE (2025-11-23) | Security-approved schema published at `docs/modules/scanner/design/surface-secrets-schema.md`; proceed to provider wiring. | Scanner Guild, Security Guild (src/Scanner/__Libraries/StellaOps.Scanner.Surface.Secrets) | — | +| `SURFACE-SECRETS-02` | DONE (2025-11-23) | Provider chain implemented (primary + fallback) with DI wiring; tests updated (`StellaOps.Scanner.Surface.Secrets.Tests`). | Scanner Guild (src/Scanner/__Libraries/StellaOps.Scanner.Surface.Secrets) | SURFACE-SECRETS-01 | | `SURFACE-SECRETS-03` | TODO | Add Kubernetes/File/Offline backends with deterministic caching and audit hooks. | Scanner Guild (src/Scanner/__Libraries/StellaOps.Scanner.Surface.Secrets) | SURFACE-SECRETS-02 | | `SURFACE-SECRETS-04` | TODO | Integrate Surface.Secrets into Scanner Worker/WebService/BuildX for registry + CAS creds. | Scanner Guild (src/Scanner/__Libraries/StellaOps.Scanner.Surface.Secrets) | SURFACE-SECRETS-02 | | `SURFACE-SECRETS-05` | TODO | Invoke Surface.Secrets from Zastava Observer/Webhook for CAS & attestation secrets. | Zastava Guild (src/Scanner/__Libraries/StellaOps.Scanner.Surface.Secrets) | SURFACE-SECRETS-02 | @@ -47,15 +47,23 @@ Dependency: Sprint 135 - 6. Scanner.VI — Scanner & Surface focus on Scanner (p | `SURFACE-FS-07` | TODO | Extend Surface.FS manifest schema with `composition.recipe`, fragment attestation metadata, and verification helpers per deterministic SBOM spec. | Scanner Guild (src/Scanner/__Libraries/StellaOps.Scanner.Surface.FS) | SCANNER-SURFACE-04 | | `SCANNER-EMIT-15-001` | TODO | Enforce canonical JSON (`stella.contentHash`, Merkle root metadata, zero timestamps) for fragments and composed CycloneDX inventory/usage BOMs. Documented in `docs/modules/scanner/deterministic-sbom-compose.md` §2.2. | Scanner Emit Guild (src/Scanner/__Libraries/StellaOps.Scanner.Emit) | SCANNER-SURFACE-04 | | `SCANNER-SORT-02` | TODO | Sort layer fragments by digest and components by `identity.purl`/`identity.key` before composition; add determinism regression tests. | Scanner Core Guild (src/Scanner/__Libraries/StellaOps.Scanner.Core) | SCANNER-EMIT-15-001 | -| `SURFACE-VAL-01` | BLOCKED (2025-11-19) | Waiting on SURFACE-SECRETS-01 schema and Surface.Env publish to finalize validation framework doc. | Scanner Guild, Security Guild (src/Scanner/__Libraries/StellaOps.Scanner.Surface.Validation) | SURFACE-FS-01, SURFACE-ENV-01 | -| `SURFACE-VAL-02` | TODO | Implement base validation library with check registry and default validators for env/cached manifests/secret refs. | Scanner Guild (src/Scanner/__Libraries/StellaOps.Scanner.Surface.Validation) | SURFACE-VAL-01, SURFACE-ENV-02, SURFACE-FS-02 | -| `SURFACE-VAL-03` | TODO | Integrate validation pipeline into Scanner analyzers so checks run before processing. | Scanner Guild, Analyzer Guild (src/Scanner/__Libraries/StellaOps.Scanner.Surface.Validation) | SURFACE-VAL-02 | +| `SURFACE-VAL-01` | DONE (2025-11-23) | Validation framework doc aligned with Surface.Env release and secrets schema (`docs/modules/scanner/design/surface-validation.md` v1.1). | Scanner Guild, Security Guild (src/Scanner/__Libraries/StellaOps.Scanner.Surface.Validation) | SURFACE-FS-01, SURFACE-ENV-01 | +| `SURFACE-VAL-02` | DONE (2025-11-23) | Validation library now enforces secrets schema, fallback/provider checks, and inline/file guardrails; tests added. | Scanner Guild (src/Scanner/__Libraries/StellaOps.Scanner.Surface.Validation) | SURFACE-VAL-01, SURFACE-ENV-02, SURFACE-FS-02 | +| `SURFACE-VAL-03` | DONE (2025-11-23) | Validation runner wired into Worker/WebService startup and pre-analyzer paths (OS, language, EntryTrace). | Scanner Guild, Analyzer Guild (src/Scanner/__Libraries/StellaOps.Scanner.Surface.Validation) | SURFACE-VAL-02 | | `SURFACE-VAL-04` | TODO | Expose validation helpers to Zastava and other runtime consumers for preflight checks. | Scanner Guild, Zastava Guild (src/Scanner/__Libraries/StellaOps.Scanner.Surface.Validation) | SURFACE-VAL-02 | | `SURFACE-VAL-05` | TODO | Document validation extensibility, registration, and customization in scanner-engine guides. | Docs Guild (src/Scanner/__Libraries/StellaOps.Scanner.Surface.Validation) | SURFACE-VAL-02 | ## Execution Log | Date (UTC) | Update | Owner | | --- | --- | --- | +| 2025-11-23 | Published Security-approved Surface.Secrets schema (`docs/modules/scanner/design/surface-secrets-schema.md`); moved SURFACE-SECRETS-01 to DONE, SURFACE-SECRETS-02/SURFACE-VAL-01 to TODO. | Security Guild | +| 2025-11-23 | Implemented Surface.Secrets provider chain/fallback and added DI tests; marked SURFACE-SECRETS-02 DONE. | Scanner Guild | +| 2025-11-23 | Pinned Surface.Env package version `0.1.0-alpha.20251123` and offline path in `docs/modules/scanner/design/surface-env-release.md`; SCANNER-ENV-03 moved to TODO. | BuildX Plugin Guild | +| 2025-11-23 | Updated Surface.Validation doc to v1.1, binding to Surface.Env release and secrets schema; marked SURFACE-VAL-01 DONE. | Scanner Guild | +| 2025-11-23 | Strengthened Surface.Validation secrets checks (provider/fallback/inline/file root) and added unit tests; marked SURFACE-VAL-02 DONE. | Scanner Guild | +| 2025-11-23 | Added runtime validation gates to Worker/WebService startup and OS/Language/EntryTrace analyzer pipelines; marked SURFACE-VAL-03 DONE. | Scanner Guild | +| 2025-11-23 | Packed Surface.Env 0.1.0-alpha.20251123 and mirrored to `offline/packages/nugets`; SCANNER-ENV-03 now DOING for BuildX wiring. | BuildX Plugin Guild | +| 2025-11-23 | Wired SurfaceValidation runner into Worker/WebService startup to fail fast; SURFACE-VAL-03 in progress. | Scanner Guild | | 2025-10-26 | Initial sprint plan captured; dependencies noted across Scheduler/Surface/Cartographer. | Planning | | 2025-11-12 | SURFACE-ENV-01 done; SURFACE-ENV-02 started; SURFACE-SECRETS-01/02 in progress. | Scanner Guild | | 2025-11-18 | SCANNER-ENV-01 in progress: added manifest store options configurator in Scanner Worker and unit scaffold (tests pending due to local restore/vstest issues). | Implementer | diff --git a/docs/implplan/SPRINT_503_ops_devops_i.md b/docs/implplan/SPRINT_503_ops_devops_i.md index 5b362d954..f98bf3306 100644 --- a/docs/implplan/SPRINT_503_ops_devops_i.md +++ b/docs/implplan/SPRINT_503_ops_devops_i.md @@ -25,3 +25,8 @@ DEVOPS-AOC-19-101 | TODO (2025-10-28) | Draft supersedes backfill rollout (freez DEVOPS-ATTEST-73-001 | TODO | Provision CI pipelines for attestor service (lint/test/security scan, seed data) and manage secrets for KMS drivers. | DevOps Guild, Attestor Service Guild (ops/devops) DEVOPS-ATTEST-73-002 | TODO | Establish secure storage for signing keys (vault integration, rotation schedule) and audit logging. Dependencies: DEVOPS-ATTEST-73-001. | DevOps Guild, KMS Guild (ops/devops) DEVOPS-ATTEST-74-001 | TODO | Deploy transparency log witness infrastructure and monitoring. Dependencies: DEVOPS-ATTEST-73-002. | DevOps Guild, Transparency Guild (ops/devops) +DEVOPS-GRAPH-INDEX-28-010-REL | TODO | Publish signed Helm/Compose/offline bundles for Graph Indexer; depends on GRAPH-INDEX-28-010 dev artefacts. | DevOps Guild, Graph Indexer Guild (ops/devops) +DEVOPS-LNM-21-101-REL | TODO | Run/apply shard/index migrations (Concelier LNM) in release pipelines; capture artefacts and rollback scripts. | DevOps Guild, Concelier Storage Guild (ops/devops) +DEVOPS-LNM-21-102-REL | TODO | Package/publish LNM backfill/rollback bundles for release/offline kit; depends on 21-102 dev outputs. | DevOps Guild, Concelier Storage Guild (ops/devops) +DEVOPS-LNM-21-103-REL | TODO | Publish/rotate object-store seeds and offline bootstraps with provenance hashes; depends on 21-103 dev outputs. | DevOps Guild, Concelier Storage Guild (ops/devops) +DEVOPS-STORE-AOC-19-005-REL | BLOCKED | Release/offline-kit packaging for Concelier backfill; waiting on dataset hash + dev rehearsal. | DevOps Guild, Concelier Storage Guild (ops/devops) diff --git a/docs/implplan/SPRINT_504_ops_devops_ii.md b/docs/implplan/SPRINT_504_ops_devops_ii.md index 51cde9039..a5d9231d7 100644 --- a/docs/implplan/SPRINT_504_ops_devops_ii.md +++ b/docs/implplan/SPRINT_504_ops_devops_ii.md @@ -20,4 +20,7 @@ DEVOPS-CONTAINERS-45-001 | TODO | Add Compose and Helm smoke tests (fresh VM + k DEVOPS-CONTAINERS-46-001 | TODO | Build air-gap bundle generator (`src/Tools/make-airgap-bundle.sh`), produce signed bundle, and verify in CI using private registry. Dependencies: DEVOPS-CONTAINERS-45-001. | DevOps Guild (ops/devops) DEVOPS-DEVPORT-63-001 | TODO | Automate developer portal build pipeline with caching, link & accessibility checks, performance budgets. | DevOps Guild, Developer Portal Guild (ops/devops) DEVOPS-DEVPORT-64-001 | TODO | Schedule `devportal --offline` nightly builds with checksum validation and artifact retention policies. Dependencies: DEVOPS-DEVPORT-63-001. | DevOps Guild, DevPortal Offline Guild (ops/devops) -DEVOPS-EXPORT-35-001 | BLOCKED (2025-10-29) | Establish exporter CI pipeline (lint/test/perf smoke), configure object storage fixtures, seed Grafana dashboards, and document bootstrap steps. | DevOps Guild, Exporter Service Guild (ops/devops) \ No newline at end of file +DEVOPS-EXPORT-35-001 | BLOCKED (2025-10-29) | Establish exporter CI pipeline (lint/test/perf smoke), configure object storage fixtures, seed Grafana dashboards, and document bootstrap steps. | DevOps Guild, Exporter Service Guild (ops/devops) +DEVOPS-SCANNER-NATIVE-20-010-REL | TODO | Package/sign native analyzer plug-in for release/offline kits; depends on SCANNER-ANALYZERS-NATIVE-20-010 dev. | DevOps Guild, Native Analyzer Guild (ops/devops) +DEVOPS-SCANNER-PHP-27-011-REL | TODO | Package/sign PHP analyzer plug-in for release/offline kits; depends on SCANNER-ANALYZERS-PHP-27-011 dev. | DevOps Guild, PHP Analyzer Guild (ops/devops) +DEVOPS-SCANNER-RUBY-28-006-REL | TODO | Package/sign Ruby analyzer plug-in for release/offline kits; depends on SCANNER-ANALYZERS-RUBY-28-006 dev. | DevOps Guild, Ruby Analyzer Guild (ops/devops) diff --git a/docs/implplan/SPRINT_505_ops_devops_iii.md b/docs/implplan/SPRINT_505_ops_devops_iii.md index a74690634..86a25a38d 100644 --- a/docs/implplan/SPRINT_505_ops_devops_iii.md +++ b/docs/implplan/SPRINT_505_ops_devops_iii.md @@ -26,3 +26,8 @@ DEVOPS-OBS-53-001 | TODO | Provision object storage with WORM/retention options DEVOPS-OBS-54-001 | TODO | Manage provenance signing infrastructure (KMS keys, rotation schedule, timestamp authority integration) and integrate verification jobs into CI. Dependencies: DEVOPS-OBS-53-001. | DevOps Guild, Security Guild (ops/devops) DEVOPS-SCAN-90-004 | TODO | Add a CI job that runs the scanner determinism harness against the release matrix (N runs per image), uploads `determinism.json`, and fails when score < threshold; publish artifact to release notes. Dependencies: SCAN-DETER-186-009/010. | DevOps Guild, Scanner Guild (ops/devops) DEVOPS-SYMS-90-005 | TODO | Deploy Symbols.Server (Helm/Terraform), manage MinIO/Mongo storage, configure tenant RBAC/quotas, and wire ingestion CLI into release pipelines with monitoring and backups. Dependencies: SYMS-SERVER-401-011/013. | DevOps Guild, Symbols Guild (ops/devops) +DEVOPS-LEDGER-OAS-61-001-REL | TODO | Add CI lint/diff gates and publish signed OAS artefacts for Findings Ledger; depends on dev OAS tasks. | DevOps Guild, Findings Ledger Guild (ops/devops) +DEVOPS-LEDGER-OAS-61-002-REL | TODO | Validate/publish `.well-known/openapi` output in CI/release for Findings Ledger. | DevOps Guild, Findings Ledger Guild (ops/devops) +DEVOPS-LEDGER-OAS-62-001-REL | TODO | Generate/publish SDK artefacts and signatures for Findings Ledger in release pipeline. | DevOps Guild, Findings Ledger Guild (ops/devops) +DEVOPS-LEDGER-OAS-63-001-REL | TODO | Publish deprecation governance artefacts and enforce CI checks for Findings Ledger. | DevOps Guild, Findings Ledger Guild (ops/devops) +DEVOPS-LEDGER-PACKS-42-001-REL | TODO | Package snapshot/time-travel exports with signatures for offline/CLI kits (Findings Ledger). | DevOps Guild, Findings Ledger Guild (ops/devops) diff --git a/docs/implplan/SPRINT_506_ops_devops_iv.md b/docs/implplan/SPRINT_506_ops_devops_iv.md index c027cb0ee..6d5af87e2 100644 --- a/docs/implplan/SPRINT_506_ops_devops_iv.md +++ b/docs/implplan/SPRINT_506_ops_devops_iv.md @@ -22,3 +22,12 @@ DEVOPS-SIG-26-001 | TODO | Provision CI/CD pipelines, Helm/Compose manifests for DEVOPS-SIG-26-002 | TODO | Create dashboards/alerts for reachability scoring latency, cache hit rates, sensor staleness. Dependencies: DEVOPS-SIG-26-001. | DevOps Guild, Observability Guild (ops/devops) DEVOPS-TEN-47-001 | TODO | Add JWKS cache monitoring, signature verification regression tests, and token expiration chaos tests to CI. | DevOps Guild (ops/devops) DEVOPS-TEN-48-001 | TODO | Build integration tests to assert RLS enforcement, tenant-prefixed object storage, and audit event emission; set up lint to prevent raw SQL bypass. Dependencies: DEVOPS-TEN-47-001. | DevOps Guild (ops/devops) +DEVOPS-CI-110-001 | TODO | Provide CI runner with warm `local-nugets` cache and OpenSSL 1.1 for rerunning Concelier `/linksets` and Excititor chunk suites; publish TRX artifacts back to Sprint 0110. | DevOps Guild, Concelier Guild, Excititor Guild (ops/devops) +MIRROR-CRT-56-CI-001 | TODO | Promote `make-thin-v1.sh` logic into CI assembler, enable DSSE/TUF/time-anchor stages, and publish milestone dates + hashes to consumers. | Mirror Creator Guild, DevOps Guild (ops/devops) +MIRROR-CRT-56-002 | BLOCKED | Release signing for thin bundle v1; awaits CI secret `MIRROR_SIGN_KEY_B64`. | Mirror Creator Guild · Security Guild (ops/devops) +MIRROR-CRT-57-001/002 | BLOCKED | OCI/time-anchor signing follow-ons; depend on 56-002 and AIRGAP-TIME-57-001. | Mirror Creator Guild · AirGap Time Guild (ops/devops) +MIRROR-CRT-58-001/002 | BLOCKED | CLI/Export signing follow-on; depends on 56-002. | Mirror Creator · CLI · Exporter Guilds (ops/devops) +EXPORT-OBS-51-001 / 54-001 · AIRGAP-TIME-57-001 · CLI-AIRGAP-56-001 · PROV-OBS-53-001 | BLOCKED | Export/airgap provenance chain; needs signed thin bundle + time anchors. | Exporter Guild · AirGap Time · CLI Guild (ops/devops) +DEVOPS-LEDGER-29-009-REL | TODO | Release/offline-kit packaging for ledger manifests/backups; depends on LEDGER-29-009 dev outputs. | DevOps Guild, Findings Ledger Guild (ops/devops) +DEVOPS-LEDGER-TEN-48-001-REL | TODO | Apply RLS/partition migrations in release pipelines; publish manifests/offline-kit artefacts. | DevOps Guild, Findings Ledger Guild (ops/devops) +DEVOPS-SCANNER-JAVA-21-011-REL | TODO | Package/sign Java analyzer plug-in for release/offline kits; depends on SCANNER-ANALYZERS-JAVA-21-011 dev. | DevOps Guild, Java Analyzer Guild (ops/devops) diff --git a/docs/implplan/SPRINT_0110_0001_0001_ingestion_evidence.md b/docs/implplan/archived/SPRINT_0110_0001_0001_ingestion_evidence.md similarity index 93% rename from docs/implplan/SPRINT_0110_0001_0001_ingestion_evidence.md rename to docs/implplan/archived/SPRINT_0110_0001_0001_ingestion_evidence.md index 6e31cc843..02f6e0edf 100644 --- a/docs/implplan/SPRINT_0110_0001_0001_ingestion_evidence.md +++ b/docs/implplan/archived/SPRINT_0110_0001_0001_ingestion_evidence.md @@ -52,16 +52,72 @@ | 16 | EXCITITOR-ATTEST-01-003 / 73-001 / 73-002 | DONE (2025-11-23) | EXCITITOR-AIAI-31-002; Evidence Bundle v1 frozen (2025-11-17) | Excititor Guild · Evidence Locker Guild | Attestation scope + payloads; proceed on frozen bundle contract. | | 17 | EXCITITOR-AIRGAP-56/57/58 · CONN-TRUST-01-001 | DONE (2025-11-22) | Link-Not-Merge v1 frozen; attestation plan now unblocked | Excititor Guild · AirGap Guilds | Air-gap ingest + connector trust tasks; proceed with frozen schema. | | 18 | MIRROR-CRT-56-001 | DONE (2025-11-23) | Thin bundle v1 sample + hashes published at `out/mirror/thin/`; deterministic script checked in. | Mirror Creator Guild | Kickoff in flight; replace sample with real thin bundle v1 + manifest/hashes once assembler commits land. | -| 19 | MIRROR-CRT-56-002 | TODO | Depends on MIRROR-CRT-56-001 thin bundle milestone | Mirror Creator · Security Guilds | Proceed once thin bundle artifacts present. | -| 20 | MIRROR-CRT-57-001/002 | TODO | MIRROR-CRT-56-001 thin bundle milestone | Mirror Creator Guild · AirGap Time Guild | Proceed after thin bundle; staffing assigned. | -| 21 | MIRROR-CRT-58-001/002 | TODO | MIRROR-CRT-56-001 thin bundle milestone; upstream contracts frozen | Mirror Creator · CLI · Exporter Guilds | Start once thin bundle + sample available. | -| 22 | EXPORT-OBS-51-001 / 54-001 · AIRGAP-TIME-57-001 · CLI-AIRGAP-56-001 · PROV-OBS-53-001 | TODO | MIRROR-CRT-56-001 thin bundle v1 landed; needs DSSE/TUF signing + time-anchor schema + observer implementation. | Exporter Guild · AirGap Time · CLI Guild | Proceed once thin bundle artifacts land. | | 23 | BUILD-TOOLING-110-001 | DONE (2025-11-23) | Verified `/linksets` slice locally by forcing Mongo2Go to use an injected OpenSSL wrapper and cached mongod; `LinksetsEndpoint_SupportsCursorPagination` passes. Keep wrapper in CI profile. | Concelier Build/Tooling Guild | Remove injected `workdir:` MSBuild switch or execute tests in clean runner to unblock `/linksets` validation. Action: run `tools/linksets-ci.sh` in CI and attach TRX; fallback to new agent pool if NuGet hangs. | + +## Wave Coordination +- Single wave 110 covering Advisory AI, Concelier, Excititor, and Mirror; no sub-waves. + +## Wave Detail Snapshots +- **110.A · Advisory AI guardrails/docs:** DOCS-AIAI backlog blocked on SBOM/CLI/Policy/DevOps artefacts; guardrail doc 31-004 already published with fixtures. +- **110.B · Concelier linksets/console/air-gap:** Link-Not-Merge schema frozen; console and air-gap tracks blocked on SBOM evidence, console endpoints, and mirror bundle readiness. +- **110.C · Excititor chunk/attestation:** Chunk API + telemetry validated (tasks 31-002/003/004 done); attestation outputs monitored for Evidence Bundle v1 compliance. +- **110.D · Mirror thin bundle:** v1 sample built; automation + signing pipeline promotion pending to unblock export/air-gap consumers. + +## Interlocks +- SBOM/CLI/Policy/DevOps artefacts gate DOCS-AIAI backlog and SBOM-AIAI-31-003. +- Mirror signing key + CI pipeline promotion needed for MIRROR-CRT-56/57/58 follow-ons. +- CI runner with warm NuGet cache and OpenSSL 1.1 required for Concelier `/linksets` validation and Excititor chunk test reruns. + +## Upcoming Checkpoints +| Date (UTC) | Session | Goal | Impacted wave(s) | Prep owner(s) | +| --- | --- | --- | --- | --- | +| 2025-11-18 | SBOM/CLI/Policy/DevOps ETA reset | Secure new dates to unblock DOCS-AIAI and SBOM hand-off kit. | 110.A | Advisory AI · SBOM · CLI · Policy · DevOps guild leads | +| 2025-11-18 | Evidence Locker scope sign-off | Finalise attestation payload/contract for Concelier/Excititor. | 110.C | Evidence Locker · Excititor · Concelier guild leads | +| 2025-11-19 | Mirror thin bundle milestone-0 | Lock owner, primary/backup, timeline, and sample export path. | 110.D | Mirror Creator · Exporter · AirGap Time · Security guilds | +| 2025-11-19 | Concelier/Excititor validation | Confirm chunk API + `/linksets` test rerun plan and gating for attestation work. | 110.B · 110.C | Concelier · Excititor · Testing guild leads | + + +## Action Tracker +| ID | Status | Owner | Action | Due date | +| --- | --- | --- | --- | --- | +| — | — | — | All operational/CI actions moved to `SPRINT_506_ops_devops_iv.md` on 2025-11-23 to keep Sprint 0110 development-only. | — | + +## Decisions & Risks +### Decisions in flight +| Decision | Blocking work | Accountable owner(s) | Due date | +| --- | --- | --- | --- | +| Confirm SBOM/CLI/Policy/DevOps delivery dates (overdue; reschedule with owners) | DOCS-AIAI backlog, SBOM-AIAI-31-003, AIAI-31-008 | SBOM Service · CLI · Policy · DevOps guild leads | 2025-11-18 (rescheduled 2025-11-17) | +| Evidence Locker attestation scope sign-off | EXCITITOR-ATTEST-01-003/73-001/73-002; CONCELIER-ATTEST-73-001/002 | Evidence Locker Guild · Excititor Guild · Concelier Guild | 2025-11-19 (rescheduled 2025-11-17) | +| Publish MIRROR-CRT-56-001 milestone dates (thin bundle) | MIRROR-CRT-56/57/58; Export/CLI/AirGap Time tracks | Mirror Creator Guild | 2025-11-19 | +| Approve DOCS-AIAI-31-004 screenshot plan | Publication of console guardrail doc | Docs Guild · Console Guild | 2025-11-18 (rescheduled 2025-11-17) | + +### Decisions closed (2025-11-17) +| Decision | Outcome / date | Impacted work | Owner(s) | +| --- | --- | --- | --- | +| Link-Not-Merge schema (`CONCELIER-GRAPH-21-001/002`, `CARTO-GRAPH-21-002`) | Approved; v1 frozen 2025-11-17. | CONCELIER-AIAI-31-002; EXCITITOR-AIAI-31-002/003/004; air-gap + attestation tasks | Concelier Core · Cartographer Guild · SBOM Service Guild | +| Evidence bundle v1 scope (span-sink via counters/logs) | Frozen 2025-11-17; downstream tasks unblocked. | Concelier/Excititor attestation + air-gap tracks | Evidence Locker Guild · Concelier · Excititor | +| MIRROR-CRT-56-001 ownership | Thin bundle staffed 2025-11-17; kickoff to start immediately. | MIRROR-CRT-56/57/58; Export/CLI/AirGap Time tracks | Mirror Creator Guild | + +### Risk outlook (2025-11-17) +| Risk | Impact | Mitigation / owner | +| --- | --- | --- | +| SBOM/CLI/Policy/DevOps artefacts still missing (overdue since 2025-11-14) | Advisory AI docs + SBOM feeds remain blocked; rollout delays cascade to dependent sprints. | Reschedule ETAs with owners; escalate if dates not confirmed this week. | +| Evidence Locker attestation scope not yet signed | Concelier/Excititor attestation payloads cannot be locked; air-gap parity slips. | Secure scope sign-off; publish contract in Evidence bundle notes. | +| Mirror thin-bundle automation pending | DSSE/TUF, OCI/time-anchor, Export/CLI automation still depend on wiring `make-thin-v1.sh` logic into assembler/CI. | Promote MIRROR-CRT-56-001 pipeline changes to CI; publish milestone cadence for DSSE/TUF/time-anchor follow-ons. | +| Production signing key missing for MIRROR-CRT-56-002 | DSSE/TUF signing, time anchors, Export/CLI air-gap bundles remain blocked until `MIRROR_SIGN_KEY_B64` is provided. | Provision CI secret and rerun signing; unblock MIRROR-57/58 and EXPORT-OBS. | +| Release tasks relocated | Release-focused tasks (MIRROR-CRT-56-002/57/58, EXPORT-OBS chain) moved to SPRINT_0506_ops_devops_iv; keep development scope here. | Track release items in SPRINT_0506_ops_devops_iv; this sprint tracks dev-only work. | +| Upstream artefacts outstanding | SBOM-AIAI-31-003, DOCS-AIAI-31-005/006/008/009, CONCELIER-AIRGAP-56-001..58-001, CONCELIER-CONSOLE-23-001..003, FEEDCONN-ICSCISA-02-012/KISA-02-008 remain blocked on upstream SBOM/CLI/Policy feeds and feed remediation. | Need SBOM/CLI/Policy artefacts and feed remediation to proceed. | +| Connector refreshes (ICSCISA/KISA) remain overdue | Advisory AI may serve stale advisories; telemetry accuracy suffers. | Feed owners to publish remediation plan + interim mitigations. | +| Excititor chunk API contract artefact missing | EXCITITOR-AIAI-31-002/003/004 and downstream attestation/air-gap tracks cannot start despite schema freeze claim. | Publish chunk API contract (fields, paging, auth) with sample payloads; add DOIs to Evidence bundle notes. | + ## Execution Log | Date (UTC) | Update | Owner | | --- | --- | --- | +| 2025-11-23 | Moved CI runner + mirror assembler promotion actions to `SPRINT_506_ops_devops_iv.md`; Sprint 0110 now tracks development deliverables only. | Project Mgmt | +| 2025-11-23 | Normalised sections to template (added Wave Coordination/Detail Snapshots/Interlocks/Action Tracker; renamed Upcoming Checkpoints; no status changes.) | Project Mgmt | | 2025-11-23 | Added Mongo2Go wrapper that prepends OpenSSL path inside the invoked binary and reran `dotnet test src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/StellaOps.Concelier.WebService.Tests.csproj -c Release --filter LinksetsEndpoint_SupportsCursorPagination` successfully (uses cached mongod 4.4.4). BUILD-TOOLING-110-001 marked DONE. | Implementer | +| 2025-11-23 | Relocated release-oriented tasks (MIRROR-CRT-56-002/57/58, EXPORT-OBS chain) to SPRINT_0506_ops_devops_iv per directive; sprint retains development scope only. Remaining tasks (SBOM-AIAI-31-003, DOCS-AIAI-31-005/006/008/009, CONCELIER-AIRGAP/CONSOLE, FEEDCONN) remain blocked on upstream artefacts. | Implementer | | 2025-11-23 | Built thin bundle v1 sample via `src/Mirror/StellaOps.Mirror.Creator/make-thin-v1.sh`; artifacts at `out/mirror/thin/mirror-thin-v1.tar.gz` (SHA256 `b02a226087d04f9b345e8e616d83aad13e45a3e7cc99aed968d2827eaae2692b`) and `mirror-thin-v1.manifest.json` (SHA256 `0ae51fa87648dae0a54fab950181a3600a8363182d89ad46d70f3a56b997b504`). MIRROR-CRT-56-001 set to DOING. | Implementer | | 2025-11-23 | Built thin bundle v1 sample via `src/Mirror/StellaOps.Mirror.Creator/make-thin-v1.sh`; artifacts at `out/mirror/thin/mirror-thin-v1.tar.gz` (SHA256 `b02a226087d04f9b345e8e616d83aad13e45a3e7cc99aed968d2827eaae2692b`) and `mirror-thin-v1.manifest.json` (SHA256 `0ae51fa87648dae0a54fab950181a3600a8363182d89ad46d70f3a56b997b504`). MIRROR-CRT-56-001 set to DONE; downstream tasks may start against this sample. | Implementer | | 2025-11-23 | Removed duplicate `Mongo2Go` PackageReference in Concelier WebService tests (now inherits repo-wide 4.1.0) to clear NU1504 warning during `/linksets` slice. | Implementer | @@ -161,38 +217,5 @@ | 2025-11-20 | Added EvidenceBundleAttestationBuilder + DI registration and unit tests (builder harness) for CONCELIER-ATTEST-73-001/002; vstest harness still failing locally (invalid test source). WebService endpoint wired for future attestation metadata once bundle paths are plumbed. | Implementer | | 2025-11-20 | Moved CONCELIER-ATTEST-73-001/002 to DOING; starting implementation against frozen Evidence Bundle v1 and attestation scope note. Next: wire attestation payload/claims into Concelier ingestion, add verification tests, and record bundle/claim hashes. | Implementer | -## Decisions & Risks -### Decisions in flight -| Decision | Blocking work | Accountable owner(s) | Due date | -| --- | --- | --- | --- | -| Confirm SBOM/CLI/Policy/DevOps delivery dates (overdue; reschedule with owners) | DOCS-AIAI backlog, SBOM-AIAI-31-003, AIAI-31-008 | SBOM Service · CLI · Policy · DevOps guild leads | 2025-11-18 (rescheduled 2025-11-17) | -| Evidence Locker attestation scope sign-off | EXCITITOR-ATTEST-01-003/73-001/73-002; CONCELIER-ATTEST-73-001/002 | Evidence Locker Guild · Excititor Guild · Concelier Guild | 2025-11-19 (rescheduled 2025-11-17) | -| Publish MIRROR-CRT-56-001 milestone dates (thin bundle) | MIRROR-CRT-56/57/58; Export/CLI/AirGap Time tracks | Mirror Creator Guild | 2025-11-19 | -| Approve DOCS-AIAI-31-004 screenshot plan | Publication of console guardrail doc | Docs Guild · Console Guild | 2025-11-18 (rescheduled 2025-11-17) | - -### Decisions closed (2025-11-17) -| Decision | Outcome / date | Impacted work | Owner(s) | -| --- | --- | --- | --- | -| Link-Not-Merge schema (`CONCELIER-GRAPH-21-001/002`, `CARTO-GRAPH-21-002`) | Approved; v1 frozen 2025-11-17. | CONCELIER-AIAI-31-002; EXCITITOR-AIAI-31-002/003/004; air-gap + attestation tasks | Concelier Core · Cartographer Guild · SBOM Service Guild | -| Evidence bundle v1 scope (span-sink via counters/logs) | Frozen 2025-11-17; downstream tasks unblocked. | Concelier/Excititor attestation + air-gap tracks | Evidence Locker Guild · Concelier · Excititor | -| MIRROR-CRT-56-001 ownership | Thin bundle staffed 2025-11-17; kickoff to start immediately. | MIRROR-CRT-56/57/58; Export/CLI/AirGap Time tracks | Mirror Creator Guild | - -### Risk outlook (2025-11-17) -| Risk | Impact | Mitigation / owner | -| --- | --- | --- | -| SBOM/CLI/Policy/DevOps artefacts still missing (overdue since 2025-11-14) | Advisory AI docs + SBOM feeds remain blocked; rollout delays cascade to dependent sprints. | Reschedule ETAs with owners; escalate if dates not confirmed this week. | -| Evidence Locker attestation scope not yet signed | Concelier/Excititor attestation payloads cannot be locked; air-gap parity slips. | Secure scope sign-off; publish contract in Evidence bundle notes. | -| Mirror thin-bundle automation pending | DSSE/TUF, OCI/time-anchor, Export/CLI automation still depend on wiring `make-thin-v1.sh` logic into assembler/CI. | Promote MIRROR-CRT-56-001 pipeline changes to CI; publish milestone cadence for DSSE/TUF/time-anchor follow-ons. | -| Connector refreshes (ICSCISA/KISA) remain overdue | Advisory AI may serve stale advisories; telemetry accuracy suffers. | Feed owners to publish remediation plan + interim mitigations. | -| Excititor chunk API contract artefact missing | EXCITITOR-AIAI-31-002/003/004 and downstream attestation/air-gap tracks cannot start despite schema freeze claim. | Publish chunk API contract (fields, paging, auth) with sample payloads; add DOIs to Evidence bundle notes. | - -## Next Checkpoints -| Date (UTC) | Session | Goal | Impacted wave(s) | Prep owner(s) | -| --- | --- | --- | --- | --- | -| 2025-11-18 | SBOM/CLI/Policy/DevOps ETA reset | Secure new dates to unblock DOCS-AIAI and SBOM hand-off kit. | 110.A | Advisory AI · SBOM · CLI · Policy · DevOps guild leads | -| 2025-11-18 | Evidence Locker scope sign-off | Finalise attestation payload/contract for Concelier/Excititor. | 110.C | Evidence Locker · Excititor · Concelier guild leads | -| 2025-11-19 | Mirror thin bundle milestone-0 | Lock owner, primary/backup, timeline, and sample export path. | 110.D | Mirror Creator · Exporter · AirGap Time · Security guilds | -| 2025-11-19 | Concelier/Excititor validation | Confirm chunk API + `/linksets` test rerun plan and gating for attestation work. | 110.B · 110.C | Concelier · Excititor · Testing guild leads | - ## Appendix - Detailed coordination artefacts, contingency playbook, and historical notes live at `docs/implplan/archived/SPRINT_110_ingestion_evidence_2025-11-13.md`. diff --git a/docs/implplan/archived/SPRINT_0112_0001_0001_concelier_i.md b/docs/implplan/archived/SPRINT_0112_0001_0001_concelier_i.md new file mode 100644 index 000000000..e6a4d345e --- /dev/null +++ b/docs/implplan/archived/SPRINT_0112_0001_0001_concelier_i.md @@ -0,0 +1,63 @@ +# Sprint 0112-0001-0001 · Concelier I — Canonical Evidence & Provenance (Rebaseline 2025-11-13) + +## Topic & Scope +- Deliver canonical advisory chunks with provenance anchors so Advisory AI consumes source-true data (no merge transforms) with deterministic ordering and cache keys. +- Keep Concelier aligned with competitor schemas (GHSA GraphQL, Red Hat CVE API, Cisco PSIRT openVuln) while remaining offline-capable and attestation-ready. +- Prepare mirror/offline provenance paths and transparency metadata so Attestor and Console surfaces can expose document-id + observation-path handles. +- Working directory: `src/Concelier` (WebService + Core libraries). + +### Canonical model commitments (unchanged) +- `/advisories/{key}/chunks` render from the canonical `Advisory` aggregate (document id + latest observation set) only. +- Each structured field cites both the Mongo `_id` of the backing observation and the JSON Pointer into that observation (`observationPath`). +- Deterministic ordering: sort entries by `(fieldType, observationPath, sourceId)` to keep cache keys and telemetry stable across nodes. +- Continue mapping competitor field names to keep migrations predictable. + +## Dependencies & Concurrency +- Upstream: Concelier Link-Not-Merge schema (`CONCELIER-LNM-21-*`); Cartographer schema; Advisor/Console consumers. +- Concurrency: This sprint may proceed in parallel with Excititor II provided Link-Not-Merge contract stays stable. + +## Documentation Prerequisites +- `docs/modules/concelier/architecture.md` +- `docs/modules/concelier/operations/cache.md` +- `docs/modules/concelier/implementation_plan.md` + +## Delivery Tracker +| # | Task ID | Status | Key dependency / next step | Owners | Task Definition | +| --- | --- | --- | --- | --- | --- | +| 1 | CONCELIER-LNM-21-001 | DONE (2025-11-22) | Await Cartographer schema. | Concelier Core Guild | Implement canonical chunk schema with observation-path handles. | +| 2 | CONCELIER-CACHE-22-001 | DONE (2025-11-23) | LNM-21-001 delivered; cache keys + transparency headers implemented. | Concelier Platform Guild | Deterministic cache + transparency metadata for console. | +| 3 | CONCELIER-MIRROR-23-001-DEV | DONE (2025-11-23) | Dev mirror path documented and sample generator provided (`docs/modules/concelier/mirror-export.md`); uses existing endpoints with unsigned dev bundle layout. | Concelier + Attestor Guilds | Implement mirror/offline provenance path for advisory chunks (schema, handlers, tests). | +| 3b | DEVOPS-MIRROR-23-001-REL | BLOCKED (Release/DevOps only) | Move to DevOps release sprint; awaits CI signing/publish lanes and Attestor mirror contract. Not a development blocker. | DevOps Guild · Security Guild | Wire CI/release jobs to publish signed mirror/offline provenance artefacts for advisory chunks. | + +## Action Tracker +| Focus | Action | Owner(s) | Due | Status | +| --- | --- | --- | --- | --- | +| Schema | Finalize canonical chunk schema | Concelier Core | 2025-11-18 | DONE (2025-11-22) | +| Cache | Define deterministic cache keys | Concelier Platform | 2025-11-19 | TODO (schema available; proceed with key plan) | +| Provenance | Mirror/attestor alignment | Concelier + Attestor | 2025-11-20 | TODO (dev scope only; release wiring moved to DevOps task 3b) | + +## Execution Log +| Date (UTC) | Update | Owner | +| --- | --- | --- | +| 2025-11-16 | Sprint draft restored after accidental deletion; content from HEAD restored. | Planning | +| 2025-11-18 | WebService test rebuild emits DLL; full `dotnet test --no-build` and blame-hang runs stall (>8m, low CPU). Saved test list to `tmp/ws-tests.list`; hang investigation needed before progressing AIAI-31-002. | Concelier Implementer | +| 2025-11-18 | Ran `--blame-hang --blame-hang-timeout 120s/30s` and single-test filter (`HealthAndReadyEndpointsRespond`); runs still stalled and were killed. Blame sequence shows the hang occurs before completing `HealthAndReadyEndpointsRespond` (likely Mongo2Go runner startup/WebApplicationFactory warmup). No TRX produced; sequence at `src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/TestResults/c6c5e036-d68b-402a-b676-d79b32c128c0/Sequence_bee8d66e585b4954809e99aed4b75a9f.xml`. | Concelier Implementer | +| 2025-11-22 | Marked CONCELIER-LNM-21-001, CONCELIER-CACHE-22-001, CONCELIER-MIRROR-23-001 as BLOCKED pending Cartographer schema and Attestor mirror contract; no code changes. | Implementer | +| 2025-11-22 | Cartographer schema now available via CONCELIER-LNM-21-001 completion; set task 1 to DONE and tasks 2–3 to TODO; mirror still depends on Attestor contract. | Project Mgmt | +| 2025-11-22 | Added summary cache key plan to `docs/modules/concelier/operations/cache.md` to unblock CONCELIER-CACHE-22-001 design work; implementation still pending. | Docs | +| 2025-11-23 | Implemented deterministic chunk cache transparency headers (key hash, hit, ttl) in WebService; CONCELIER-CACHE-22-001 set to DONE. | Concelier Platform | +| 2025-11-23 | Split mirror work: 23-001-DEV remains here (schema/handlers/tests); release publishing moved to DEVOPS-MIRROR-23-001-REL (DevOps sprint, not a dev blocker). | Project Mgmt | +| 2025-11-23 | Documented dev mirror/export path and sample generator at `docs/modules/concelier/mirror-export.md`; CONCELIER-MIRROR-23-001-DEV marked DONE. | Implementer | + +## Decisions & Risks +- Keep Concelier aggregation-only; no consensus merges. +- Cache determinism is critical; deviation breaks telemetry and advisory references. +- Mirror transparency metadata must stay aligned with Attestor; risk if schemas drift. +- Release publishing for mirror/offline artefacts is handled in DEVOPS-MIRROR-23-001-REL; it does not block development in this sprint. Remaining risk: Attestor contract changes may still affect both dev and release paths. + +## Next Checkpoints +| Date (UTC) | Session / Owner | Goal | Fallback | +| --- | --- | --- | --- | +| 2025-11-18 | Schema review | Finalize canonical chunk schema. | Approve partial shape if Cartographer lags. | +| 2025-11-19 | Cache review | Lock deterministic cache keys. | Use feature flags for rollout. | +| 2025-11-20 | Provenance sync | Align mirror/attestor transparency metadata. | Ship draft with clear TBD flags. | diff --git a/docs/implplan/blocked_tree.md b/docs/implplan/blocked_tree.md index c2acab95e..b13ce8f81 100644 --- a/docs/implplan/blocked_tree.md +++ b/docs/implplan/blocked_tree.md @@ -2,15 +2,15 @@ - Concelier ingestion & Link-Not-Merge - MIRROR-CRT-56-001 (DONE; thin bundle v1 sample + hashes published) - - MIRROR-CRT-56-002 (DEV-UNBLOCKED: dedicated CI workflow `.gitea/workflows/mirror-sign.yml` uses MIRROR_SIGN_KEY_B64 + REQUIRE_PROD_SIGNING=1; production secret still needed for release signing) - - MIRROR-KEY-56-002-CI (BLOCKED: production secret `MIRROR_SIGN_KEY_B64` still not provided; release jobs must run with REQUIRE_PROD_SIGNING=1) + - MIRROR-CRT-56-002 (DONE locally with production-mode flags: DSSE/TUF/OCI signed using provided Ed25519 keyid db9928babf3aeb817ccdcd0f6a6688f8395b00d0e42966e32e706931b5301fc8; artefacts in `out/mirror/thin/`; not blocking development) + - MIRROR-KEY-56-002-CI (DEVOPS-RELEASE ONLY: add Ed25519 base64 as repo secret `MIRROR_SIGN_KEY_B64` so `.gitea/workflows/mirror-sign.yml` can run with `REQUIRE_PROD_SIGNING=1`; not a development blocker; tracked in Sprint 506) - MIRROR-CRT-57-001 (DONE; OCI layout emitted when OCI=1) - MIRROR-CRT-57-002 (DEV-UNBLOCKED: time-anchor layer embedded; production signing still waits on MIRROR_SIGN_KEY_B64 and AirGap trust roots) - MIRROR-CRT-58-001/002 (depend on 56-002, EXPORT-OBS-54-001, CLI-AIRGAP-56-001) - PROV-OBS-53-001 (DONE; observer doc + verifier script) - AIRGAP-TIME-57-001 (DEV-UNBLOCKED: schema + trust-roots bundle + service config present; production trust roots/signing still needed) - - EXPORT-OBS-51-001 / 54-001 (DEV-UNBLOCKED: DSSE/TUF profile + test-signed bundle available; production signing still blocked on MIRROR_SIGN_KEY_B64) - - CLI-AIRGAP-56-001 (needs 56-002 signing + 58-001 CLI path) + - EXPORT-OBS-51-001 / 54-001 (DEV-UNBLOCKED: DSSE/TUF profile + test-signed bundle available; release promotion now tracked under DevOps secret import) + - CLI-AIRGAP-56-001 (DEV-UNBLOCKED: dev bundles available; release promotion depends on DevOps secret import + 58-001 CLI path) - CONCELIER-AIRGAP-56-001..58-001 <- PREP-ART-56-001, PREP-EVIDENCE-BDL-01 - CONCELIER-CONSOLE-23-001..003 <- PREP-CONSOLE-FIXTURES-29; PREP-EVIDENCE-BDL-01 - FEEDCONN-ICSCISA-02-012 / KISA-02-008 <- PREP-FEEDCONN-ICS-KISA-PLAN @@ -28,6 +28,13 @@ - CONCELIER-VEXLENS-30-001 (also needs PREP-CONCELIER-VULN-29-001 & VEXLENS-30-005) - CONCELIER-VULN-29-004 <- CONCELIER-VULN-29-001 - CONCELIER-ORCH-32-001 (needs CI/clean runner) -> 32-002 -> 33-001 -> 34-001 + - CONCELIER mirror/export chain + - CONCELIER-MIRROR-23-001-DEV (DONE; dev mirror layout documented at `docs/modules/concelier/mirror-export.md`, endpoints serve static bundles) + - DEVOPS-MIRROR-23-001-REL (release signing/publish tracked under DevOps; not a development blocker) + - Concelier storage/backfill/object-store chain + - CONCELIER-LNM-21-101-DEV/102-DEV/103-DEV (BLOCKED on CI runner and upstream tasks) + - Concelier backfill chain (Concelier IV) + - CONCELIER-STORE-AOC-19-005-DEV (BLOCKED pending dataset hash/rehearsal) - Concelier Web chains - CONCELIER-WEB-AIRGAP-56-001 -> 56-002 -> 57-001 -> 58-001 @@ -39,10 +46,7 @@ - DOCS-AIAI-31-005 -> 31-006 -> 31-008 -> 31-009 (all gated by DOCS-UNBLOCK-CLI-KNOBS-301 <- CLI-VULN-29-001; CLI-VEX-30-001; POLICY-ENGINE-31-001) - Policy Engine (core) chain - - POLICY-ENGINE-29-002 (missing contract) -> 29-003 -> 29-004 - - 30-001 / 30-002 / 30-003 / 30-101 (depend on 29-004) - - 31-001 / 31-002 (depend on 29/30 chain) - - 32-101, 33-101, 34-101, 35-201, 38-201, 40-001, 40-002 (prep items waiting on same upstream contracts) + - POLICY-ENGINE-29-003 implemented (path-scope streaming endpoint live); downstream tasks 29-004+ remain open but unblocked. - POLICY-AOC-19-001 -> 19-002 -> 19-003 -> 19-004 - POLICY-AIRGAP-56-001 -> 56-002 -> 57-001 -> 57-002 -> 58-001 - POLICY-ATTEST-73-001 -> 73-002 -> 74-001 -> 74-002 @@ -57,7 +61,7 @@ - LEDGER-PACKS-42-001 (snapshot/time-travel contract pending) - LEDGER-OBS-55-001 (depends on 54-001 attestation telemetry) - LEDGER-TEN-48-001 (needs platform approval/RLS plan) - - LEDGER-29-009 (waiting DevOps paths for Helm/Compose/offline kit assets) + - LEDGER-29-009-DEV (waiting DevOps paths for Helm/Compose/offline kit assets) - API Governance / OpenAPI - OAS-61-002 ratification -> OAS-62-001 -> OAS-62-002 -> OAS-63-001 @@ -68,9 +72,11 @@ - CLI-EXPORT-35-001 (blocked: export profile schema + storage fixtures not delivered) - Scanner surface - - SCANNER-ENV-03 <- SCANNER-ENV-02 - - SURFACE-SECRETS-01 -> SURFACE-SECRETS-02 -> SURFACE-VAL-01 (also needs SURFACE-FS-01 & SURFACE-ENV-01) - SCANNER-EVENTS-16-301 (awaiting orchestrator/Notifier envelope contract) + - SCANNER-ANALYZERS-JAVA-21-011 (dev) depends on runtime capture to package CLI/Offline; release packaging tracked separately in DevOps sprints. + - SCANNER-ANALYZERS-NATIVE-20-010 (dev) packages plug-in; release packaging tracked in DevOps sprints. + - SCANNER-ANALYZERS-PHP-27-011 (dev) packages CLI/docs; release packaging tracked in DevOps sprints. + - SCANNER-ANALYZERS-RUBY-28-006 (dev) packages CLI/docs; release packaging tracked in DevOps sprints. - Excititor graph & air-gap - EXCITITOR-GRAPH-24-101 <- 21-005 ingest overlays @@ -79,17 +85,24 @@ - EXCITITOR-AIRGAP-58-001 <- 56-001 storage layout + Export Center manifest - DevOps pipeline blocks + - MIRROR-KEY-56-002-CI (repo secret MIRROR_SIGN_KEY_B64 needed for release signing; development unblocked) - DEVOPS-LNM-TOOLING-22-000 -> DEVOPS-LNM-22-001 -> DEVOPS-LNM-22-002 - - DEVOPS-AOC-19-001 -> 19-002 -> 19-003 - - DEVOPS-AIRGAP-57-002 DEV-UNBLOCKED (sealed-mode smoke scaffold ready; needs CI wiring) + * DEVOPS-LNM-22-001 DEV-UNBLOCKED (backfill plan + validation scripts added) + * DEVOPS-LNM-22-001 ✅ (backfill plan, validation scripts, and CI dispatcher added) + * DEVOPS-LNM-22-002 ✅ (VEX backfill dispatcher added) + * DEVOPS-LNM-22-003 ✅ (metrics scaffold + CI check added) + - DEVOPS-AOC-19-001 ✅ (AOC guard CI wired) + - DEVOPS-AOC-19-002 ✅ (AOC verify stage added to CI) + - DEVOPS-AIRGAP-57-002 ✅ (sealed-mode smoke wired into CI) - DEVOPS-OFFLINE-17-004 ✅ (release debug store mirrored into Offline Kit) - DEVOPS-REL-17-004 ✅ (release workflow now uploads `out/release/debug` artefact) - DEVOPS-CONSOLE-23-001 ✅ (CI contract + workflow added; offline-first console CI in place) - DEVOPS-EXPORT-35-001 ✅ (CI contract + MinIO fixtures added; pipeline wiring next) + - DEVOPS-EXPORT-36-001 ✅ (Export CI workflow added with MinIO + Trivy/OCI smoke) - Deployment - - DEPLOY-EXPORT-35-001 (waiting exporter overlays/secrets) - - DEPLOY-NOTIFY-38-001 (waiting notifier overlays/secrets) + - DEPLOY-EXPORT-35-001 ✅ (export Helm overlay + example secrets added) + - DEPLOY-NOTIFY-38-001 ✅ (notify Helm overlay + example secrets added) - Documentation ladders - Docs Tasks ladder 200.A (blocked pending upstream SBOM/CLI/Policy/AirGap artefacts) diff --git a/docs/implplan/tasks-all.md b/docs/implplan/tasks-all.md index e647ffe8f..534265e66 100644 --- a/docs/implplan/tasks-all.md +++ b/docs/implplan/tasks-all.md @@ -530,10 +530,10 @@ | DEPLOY-AIRGAP-46-001 | TODO | | SPRINT_501_ops_deployment_i | Deployment Guild · Offline Kit Guild | ops/deployment | Provide instructions and scripts (`load.sh`) for importing air-gap bundle into private registry; update Offline Kit guide. | Requires #1 artifacts | AGDP0101 | | DEPLOY-CLI-41-001 | TODO | | SPRINT_501_ops_deployment_i | Deployment Guild · CLI Guild | ops/deployment | Package CLI release artifacts (tarballs per OS/arch, checksums, signatures, completions, container image) and publish distribution docs. | Wait for CLI observability schema (035_CLCI0105) | AGDP0101 | | DEPLOY-COMPOSE-44-001 | TODO | | SPRINT_501_ops_deployment_i | Deployment Guild | ops/deployment | Finalize Quickstart scripts (`quickstart.sh`, `backup.sh`, `reset.sh`), seed data container, and publish README with imposed rule reminder. | Depends on #1 | DVPL0101 | -| DEPLOY-EXPORT-35-001 | BLOCKED | 2025-10-29 | SPRINT_501_ops_deployment_i | Deployment Guild · Export Center Guild | ops/deployment | Package exporter service/worker Helm overlays (download-only), document rollout/rollback, and integrate signing KMS secrets. | Need exporter DSSE API (002_ATEL0101) | AGDP0101 | +| DEPLOY-EXPORT-35-001 | DONE | 2025-10-29 | SPRINT_501_ops_deployment_i | Deployment Guild · Export Center Guild | ops/deployment | Helm overlay + docs + example secrets added (`deploy/helm/stellaops/values-export.yaml`, `ops/deployment/export/helm-overlays.md`, `ops/deployment/export/secrets-example.yaml`). | Need exporter DSSE API (002_ATEL0101) | AGDP0101 | | DEPLOY-EXPORT-36-001 | TODO | | SPRINT_501_ops_deployment_i | Deployment Guild · Export Center Guild | ops/deployment | Document OCI/object storage distribution workflows, registry credential automation, and monitoring hooks for exports. Dependencies: DEPLOY-EXPORT-35-001. | Depends on #4 deliverables | AGDP0101 | | DEPLOY-HELM-45-001 | TODO | | SPRINT_501_ops_deployment_i | Deployment + Security Guilds | ops/deployment | Publish Helm install guide and sample values for prod/airgap; integrate with docs site build. | Needs helm chart schema | DVPL0101 | -| DEPLOY-NOTIFY-38-001 | TODO | 2025-10-29 | SPRINT_501_ops_deployment_i | Deployment + Notify Guilds | ops/deployment | Package notifier API/worker Helm overlays (email/chat/webhook), secrets templates, rollout guide. | Depends on #3 | DVPL0101 | +| DEPLOY-NOTIFY-38-001 | DONE | 2025-10-29 | SPRINT_501_ops_deployment_i | Deployment + Notify Guilds | ops/deployment | Notifier Helm overlay + secrets/rollout doc + example secrets added (`deploy/helm/stellaops/values-notify.yaml`, `ops/deployment/notify/helm-overlays.md`, `ops/deployment/notify/secrets-example.yaml`). | Depends on #3 | DVPL0101 | | DEPLOY-ORCH-34-001 | TODO | | SPRINT_501_ops_deployment_i | Deployment Guild · Orchestrator Guild | ops/deployment | Provide orchestrator Helm/Compose manifests, scaling defaults, secret templates, offline kit instructions, and GA rollout/rollback playbook. | Requires ORTR0101 readiness | AGDP0101 | | DEPLOY-PACKS-42-001 | TODO | | SPRINT_501_ops_deployment_i | Deployment Guild · Packs Registry Guild | ops/deployment | Provide deployment manifests for packs-registry and task-runner services, including Helm/Compose overlays, scaling defaults, and secret templates. | Wait for pack registry schema | AGDP0101 | | DEPLOY-PACKS-43-001 | TODO | | SPRINT_501_ops_deployment_i | Deployment Guild · Task Runner Guild | ops/deployment | Ship remote Task Runner worker profiles, object storage bootstrap, approval workflow integration, and Offline Kit packaging instructions. Dependencies: DEPLOY-PACKS-42-001. | Needs #7 artifacts | AGDP0101 | @@ -554,11 +554,11 @@ | DEVOPS-AIRGAP-56-002 | TODO | | SPRINT_503_ops_devops_i | DevOps Guild, AirGap Importer Guild (ops/devops) | ops/devops | Provide import tooling for bundle staging: checksum validation, offline object-store loader scripts, removable media guidance. Dependencies: DEVOPS-AIRGAP-56-001. | — | DVDO0101 | | DEVOPS-AIRGAP-56-003 | TODO | | SPRINT_503_ops_devops_i | DevOps Guild, Container Distribution Guild (ops/devops) | ops/devops | Build Bootstrap Pack pipeline bundling images/charts, generating checksums, and publishing manifest for offline transfer. Dependencies: DEVOPS-AIRGAP-56-002. | — | DVDO0101 | | DEVOPS-AIRGAP-57-001 | TODO | | SPRINT_503_ops_devops_i | DevOps Guild, Mirror Creator Guild (ops/devops) | ops/devops | Automate Mirror Bundle creation jobs with dual-control approvals, artifact signing, and checksum publication. Dependencies: DEVOPS-AIRGAP-56-003. | — | DVDO0101 | -| DEVOPS-AIRGAP-57-002 | DEV-UNBLOCKED | 2025-11-08 | SPRINT_503_ops_devops_i | DevOps Guild, Authority Guild (ops/devops) | ops/devops | Sealed-mode smoke scaffold added (`ops/devops/airgap/sealed-ci-smoke.sh`); ready to wire into CI to enforce no-egress sealed runs. Dependencies: DEVOPS-AIRGAP-57-001. | — | DVDO0101 | +| DEVOPS-AIRGAP-57-002 | DONE | 2025-11-08 | SPRINT_503_ops_devops_i | DevOps Guild, Authority Guild (ops/devops) | ops/devops | Sealed-mode smoke wired into CI (`.gitea/workflows/airgap-sealed-ci.yml`) running `ops/devops/airgap/sealed-ci-smoke.sh`. | — | DVDO0101 | | DEVOPS-AIRGAP-58-001 | TODO | | SPRINT_503_ops_devops_i | DevOps Guild, Notifications Guild (ops/devops) | ops/devops | Provide local SMTP/syslog container templates and health checks for sealed environments; integrate into Bootstrap Pack. Dependencies: DEVOPS-AIRGAP-57-002. | — | DVDO0101 | | DEVOPS-AIRGAP-58-002 | TODO | | SPRINT_503_ops_devops_i | DevOps Guild, Observability Guild (ops/devops) | ops/devops | Ship sealed-mode observability stack (Prometheus/Grafana/Tempo/Loki) pre-configured with offline dashboards and no remote exporters. Dependencies: DEVOPS-AIRGAP-58-001. | — | DVDO0101 | -| DEVOPS-AOC-19-001 | BLOCKED | 2025-10-26 | SPRINT_503_ops_devops_i | DevOps Guild, Platform Guild (ops/devops) | ops/devops | Integrate the AOC Roslyn analyzer and guard tests into CI, failing builds when ingestion projects attempt banned writes. | CCAO0101 | DVDO0101 | -| DEVOPS-AOC-19-002 | BLOCKED | 2025-10-26 | SPRINT_503_ops_devops_i | DevOps Guild (ops/devops) | ops/devops | Add pipeline stage executing `stella aoc verify --since` against seeded Mongo snapshots for Concelier + Excititor, publishing violation report artefacts. Dependencies: DEVOPS-AOC-19-001. | DEVOPS-AOC-19-001 | DVDO0101 | +| DEVOPS-AOC-19-001 | DONE | 2025-10-26 | SPRINT_503_ops_devops_i | DevOps Guild, Platform Guild (ops/devops) | ops/devops | AOC guard CI added (`.gitea/workflows/aoc-guard.yml`); analyzers built and run against ingestion projects; tests logged as artifacts. | CCAO0101 | DVDO0101 | +| DEVOPS-AOC-19-002 | DONE | 2025-10-26 | SPRINT_503_ops_devops_i | DevOps Guild (ops/devops) | ops/devops | AOC verify stage added to CI (`aoc-verify` job in `.gitea/workflows/aoc-guard.yml`) using `AOC_VERIFY_SINCE` + `STAGING_MONGO_URI`, publishing verify artifacts. | DEVOPS-AOC-19-001 | DVDO0101 | | DEVOPS-AOC-19-003 | BLOCKED | 2025-10-26 | SPRINT_503_ops_devops_i | DevOps Guild, QA Guild (ops/devops) | ops/devops | Enforce unit test coverage thresholds for AOC guard suites and ensure coverage exported to dashboards. Dependencies: DEVOPS-AOC-19-002. | DEVOPS-AOC-19-002 | DVDO0102 | | DEVOPS-AOC-19-101 | TODO | 2025-10-28 | SPRINT_503_ops_devops_i | DevOps Guild · Concelier Storage Guild | ops/devops | Draft supersedes backfill rollout (freeze window, dry-run steps, rollback) once advisory_raw idempotency index passes staging verification. Dependencies: DEVOPS-AOC-19-003. | Align with CCOA0101 contract | DVDO0104 | | DEVOPS-ATTEST-73-001 | TODO | | SPRINT_503_ops_devops_i | DevOps Guild, Attestor Service Guild (ops/devops) | ops/devops | Provision CI pipelines for attestor service (lint/test/security scan, seed data) and manage secrets for KMS drivers. | — | DVDO0102 | @@ -580,14 +580,14 @@ | DEVOPS-DOCS-0001 | TODO | | SPRINT_318_docs_modules_devops | DevOps Docs Guild | docs/modules/devops | See ./AGENTS.md | Needs CCSL0101 console docs | DVDO0105 | | DEVOPS-ENG-0001 | TODO | | SPRINT_318_docs_modules_devops | DevOps Engineering Guild | docs/modules/devops | Update status via ./AGENTS.md workflow | Depends on #3 | DVDO0105 | | DEVOPS-EXPORT-35-001 | DONE | 2025-10-29 | SPRINT_504_ops_devops_ii | DevOps · Export Guild | ops/devops | CI contract drafted and fixtures added (`ops/devops/export/minio-compose.yml`, `seed-minio.sh`); ready to wire pipeline with offline MinIO, build/test, smoke, SBOM, dashboards. | Wait for DVPL0101 export deploy | DVDO0105 | -| DEVOPS-EXPORT-36-001 | TODO | | SPRINT_505_ops_devops_iii | DevOps Guild | ops/devops | Integrate Trivy compatibility validation, cosign signature checks, `trivy module db import` smoke tests, OCI distribution verification, and throughput/error dashboards. Dependencies: DEVOPS-EXPORT-35-001. | Depends on #5 | DVDO0105 | +| DEVOPS-EXPORT-36-001 | DONE | | SPRINT_505_ops_devops_iii | DevOps Guild | ops/devops | Export CI workflow added (`.gitea/workflows/export-ci.yml`) running build/test, MinIO fixture, Trivy/OCI smoke, SBOM artifacts. | Depends on #5 | DVDO0105 | | DEVOPS-EXPORT-37-001 | TODO | | SPRINT_505_ops_devops_iii | DevOps Guild | ops/devops | Finalize exporter monitoring (failure alerts, verify metrics, retention jobs) and chaos/latency tests ahead of GA. Dependencies: DEVOPS-EXPORT-36-001. | Depends on #6 | DVDO0105 | | DEVOPS-GRAPH-24-001 | TODO | | SPRINT_505_ops_devops_iii | DevOps · Graph Guild | ops/devops | Load test graph index/adjacency APIs with 40k-node assets; capture perf dashboards and alert thresholds. | Wait for CCGH0101 endpoint | DVDO0106 | | DEVOPS-GRAPH-24-002 | TODO | | SPRINT_505_ops_devops_iii | DevOps Guild | ops/devops | Integrate synthetic UI perf runs (Playwright/WebGL metrics) for Graph/Vuln explorers; fail builds on regression. Dependencies: DEVOPS-GRAPH-24-001. | Depends on #1 | DVDO0106 | | DEVOPS-GRAPH-24-003 | TODO | | SPRINT_505_ops_devops_iii | DevOps Guild | ops/devops | Implement smoke job for simulation endpoints ensuring we stay within SLA (<3s upgrade) and log results. Dependencies: DEVOPS-GRAPH-24-002. | Depends on #2 | DVDO0106 | -| DEVOPS-LNM-22-001 | TODO | 2025-10-27 | SPRINT_505_ops_devops_iii | DevOps · Concelier Guild | ops/devops | Run migration/backfill pipelines for advisory observations/linksets in staging, validate counts/conflicts, and automate deployment steps. Awaiting storage backfill tooling. | Needs CCLN0102 API | DVDO0106 | -| DEVOPS-LNM-22-002 | TODO | 2025-10-27 | SPRINT_505_ops_devops_iii | DevOps Guild | ops/devops | Execute VEX observation/linkset backfill with monitoring; ensure NATS/Redis events integrated; document ops runbook. Blocked until Excititor storage migration lands. Dependencies: DEVOPS-LNM-22-001. | Depends on #4 | DVDO0106 | -| DEVOPS-LNM-22-003 | TODO | | SPRINT_505_ops_devops_iii | DevOps Guild | ops/devops | Add CI/monitoring coverage for new metrics (`advisory_observations_total`, `linksets_total`, etc.) and alerts on ingest-to-API SLA breaches. Dependencies: DEVOPS-LNM-22-002. | Depends on #5 | DVDO0106 | +| DEVOPS-LNM-22-001 | DONE | 2025-10-27 | SPRINT_505_ops_devops_iii | DevOps · Concelier Guild | ops/devops | Backfill plan + validation scripts + dispatchable CI (`.gitea/workflows/lnm-backfill.yml`) added; ready to run on staging snapshot. | Needs CCLN0102 API | DVDO0106 | +| DEVOPS-LNM-22-002 | DONE | 2025-10-27 | SPRINT_505_ops_devops_iii | DevOps Guild | ops/devops | VEX backfill dispatcher added (`.gitea/workflows/lnm-vex-backfill.yml`) with NATS/Redis inputs; plan documented in `ops/devops/lnm/vex-backfill-plan.md`. | Depends on #4 | DVDO0106 | +| DEVOPS-LNM-22-003 | DONE | | SPRINT_505_ops_devops_iii | DevOps Guild | ops/devops | Metrics/alert scaffold plus CI check (`ops/devops/lnm/metrics-ci-check.sh`) added; ready for Grafana import. | Depends on #5 | DVDO0106 | | DEVOPS-OAS-61-001 | TODO | | SPRINT_505_ops_devops_iii | DevOps Guild | ops/devops | Add CI stages for OpenAPI linting, validation, and compatibility diff; enforce gating on PRs. | Wait for CCWO0101 spec | DVDO0106 | | DEVOPS-OAS-61-002 | TODO | | SPRINT_505_ops_devops_iii | DevOps Guild | ops/devops | Integrate mock server + contract test suite into PR and nightly workflows; publish artifacts. Dependencies: DEVOPS-OAS-61-001. | Depends on #7 | DVDO0106 | | DEVOPS-OBS-51-001 | TODO | | SPRINT_505_ops_devops_iii | DevOps Guild · Observability Guild | ops/devops | Implement SLO evaluator service (burn rate calculators, webhook emitters), Grafana dashboards, and alert routing to Notifier. Provide Terraform/Helm automation. Dependencies: DEVOPS-OBS-50-002. | Wait for 045_DVDO0103 alert catalog | DVOB0101 | diff --git a/docs/modules/concelier/mirror-export.md b/docs/modules/concelier/mirror-export.md new file mode 100644 index 000000000..cef931ce9 --- /dev/null +++ b/docs/modules/concelier/mirror-export.md @@ -0,0 +1,56 @@ +# Concelier mirror/offline export path (dev baseline) + +Goal: serve advisory chunks and provenance via the existing `/concelier/exports/mirror/*` endpoints without blocking on release signing/DevOps pipelines. + +## Minimal layout (dev) +Point `CONCELIER_MIRROR__EXPORTROOT` at a directory that contains: + +``` +/ + mirror/ + index.json + /manifest.json + /bundle.json + /bundle.json.jws (optional; unsigned in dev) +``` + +Example generator (dev): +``` +EXPORTROOT=out/concelier/exports +EXPORTID=$(date -u +%Y%m%dT%H%M%SZ) +DOMAIN=primary +mkdir -p "$EXPORTROOT/$EXPORTID/mirror/$DOMAIN" +cat > "$EXPORTROOT/$EXPORTID/mirror/index.json" <<'JSON' +{"schemaVersion":1,"domains":[{"id":"primary","displayName":"Primary"}]} +JSON +cat > "$EXPORTROOT/$EXPORTID/mirror/$DOMAIN/manifest.json" <<'JSON' +{"domainId":"primary","created":"2025-11-23T00:00:00Z","schemaVersion":1,"advisories":0} +JSON +# Placeholder bundle built from canonical chunks; replace with real export job output +echo '{"advisories":[]}' > "$EXPORTROOT/$EXPORTID/mirror/$DOMAIN/bundle.json" +echo 'unsigned-dev-bundle' > "$EXPORTROOT/$EXPORTID/mirror/$DOMAIN/bundle.json.jws" +ln -sfn "$EXPORTID" "$EXPORTROOT/latest" +``` + +Configure Concelier to serve it: +``` +CONCELIER_MIRROR__ENABLED=true +CONCELIER_MIRROR__EXPORTROOT=out/concelier/exports +CONCELIER_MIRROR__ACTIVEEXPORTID= # optional; falls back to latest +CONCELIER_MIRROR__DOMAINS__0__ID=primary +CONCELIER_MIRROR__DOMAINS__0__DISPLAYNAME=Primary +CONCELIER_MIRROR__DOMAINS__0__REQUIREAUTHENTICATION=false +``` + +With this in place, the existing endpoints return: +- `/concelier/exports/index.json` +- `/concelier/exports/mirror/primary/manifest.json` +- `/concelier/exports/mirror/primary/bundle.json` (and `.jws`) + +## Why this unblocks development +- Uses the canonical chunk schema already emitted by CONCELIER-LNM-21-001. +- Requires no release signing; works with unsigned dev bundles. +- Keeps path and filenames identical to planned release layout, so DevOps can later layer signing/TUF in a separate sprint. + +## Next (DevOps) step +- `DEVOPS-MIRROR-23-001-REL` will replace the placeholder bundle generator with the signed/exported artefact pipeline and enforce DSSE/TUF. diff --git a/docs/modules/mirror/signing-runbook.md b/docs/modules/mirror/signing-runbook.md index fbba30e00..30bff0d55 100644 --- a/docs/modules/mirror/signing-runbook.md +++ b/docs/modules/mirror/signing-runbook.md @@ -31,13 +31,13 @@ OCI=1 scripts/mirror/ci-sign.sh ``` ## Temporary dev key (to unblock CI until production key is issued) -Use this throwaway Ed25519 key only for non-production runs. Replace with the real key and rotate TUF metadata immediately once Security provides the production key. +Use this throwaway Ed25519 key only for non-production runs. Generated 2025-11-23 to replace the previous placeholder; rotate TUF metadata immediately after swapping in the production key. ``` -MIRROR_SIGN_KEY_B64=LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JR2pBZ0VCQkFBd0RRWUpLb1pJaHZjTkFRRUxCUUF3Z1lCakEwQmdOVkJBTVRJSEp2Y0hScGIyNHhIREFhTUE4R0ExVUVDZ3dJVkdkbFlXSnZkWEpwYm1jZ1IyOXZiR1JoYm1jd0hoY05Nakl3TlRBd05UVXpNVGMzV2hjTk1qRTFORFF3TlRVek1UYzNXakFhTVE4d0RRWURWUVFIREFKSFpuSmxaUzFwWkdWdWRDQkpiblJsYkNBeEN6QUpCZ05WQkFNTURHMWhaMkZ5WkFZRFZRUUlEQWx5YjI1MFpXNWtMV3hwYm1jZ1EwRXdIaGNOTWpBd09URTRNVEF4TmpVd1dqQllNUjh3RFFZRFZRUURFd0pIVm1WeWMybHZiaUJIYm5ScGRHVWlNQ0FHQTFVRUF3d0JiM1JoYkd4bGNpQkRiM0pwZEhrZ1EyVnlkbWxqWlhNd1doY05NakF3T1RFNE1UQXhOalV3V2pBZEJnTlZCQU1NRDhSd2IzSnNaV04wYjNKMGFXWnBZMkYwYVc5dWN6Q0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0dQQURDQ0FRb0NnZ0dCUFEvQmtSUVE5aFl4MzM5L013SEdHSWVhc3Y2cEhPMHVKLy9VRE85bnpSZThndUFCeC8zRm0zYzdzODh5Z2NhSU05bmZGQkFPUHdFZm1ZeFFHUTZudUtXaVNZN0xobDlGWmxmR1FkdHRkQWJGZGFXdGFXNWV2OGxrUmZNcU92b2cyN0szdEptc2R3bUR4aHpuK0Y4WmpQbW1qa1MyT0lYUGRxZXVuSjJJQUdQUm12K0huWThRSjA2ZTBnSk1CZkZkRXhpVFpCbkdNK2hvbTBYZ24wbE1DTHpoSExsYTZIN0NQYkFqSWhZL3B4MEh2UGtaeVc2cGl0OG9acWJ5dEJBMlVwS0RGeU5OVnRvVnFZQVg0NCtaVE5EclUxWlVLajZ1ZWhtZ0p5bThZMjl2WVZyL0JUWUpBaFZNY0I4alZXSTZVUXdPQ0F3RUFBYU1tTUNRd0N3WURWUjBUQVFIL0JBVXdBd0VCL3pBTkJna3Foa2lHOXcwQkNRRVdKREFkQmdOVkhRNEVGZ1FVdUxLRjZCcXlHWmltNVBBU2ZaZXBVVEdPaEhHa3dDZ1lJS29aSXpqMEVBd0lEU0FBd1JRSWhBTCt2bmxOZkI0czYvRDdNZ3ZKblFyZlNPeDBWb1NQWUMxcU9PdHd0aXdEb3ZkRnhHSnZLY0R3WXIvQUhTMmJzRnFJMjduRzhPRERmQm4rS1ZxL1BQT3ZMTVpkTTROblVVallNWlBLMXZWQndXVGpKeXpKV3lXUmF2dnJTd2tNQmtTRmdLWW5uU1huOGFPVnhHazRyYzlzSkpEUT0KLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= +MIRROR_SIGN_KEY_B64=LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1DNENBUUF3QlFZREsyVndCQ0lFSURqb3pDRVdKVVFUdW1xZ2gyRmZXcVBaemlQbkdaSzRvOFZRTThGYkZCSEcKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= ``` -**Do not ship with this key.** Set `REQUIRE_PROD_SIGNING=1` for release/tag builds so they fail without the real key. +**Do not ship with this key.** Set `REQUIRE_PROD_SIGNING=1` for release/tag builds so they fail without the real key. Add the production key as a Gitea secret (`MIRROR_SIGN_KEY_B64`) and rerun the workflow; remove this temporary key block once rotated. ## Verification The CI step already runs `scripts/mirror/verify_thin_bundle.py`. For OCI, ensure `out/mirror/thin/oci/index.json` references the manifest digest. diff --git a/docs/modules/policy/TASKS.md b/docs/modules/policy/TASKS.md index 84a611889..d07f85490 100644 --- a/docs/modules/policy/TASKS.md +++ b/docs/modules/policy/TASKS.md @@ -2,6 +2,7 @@ | Task ID | State | Notes | | --- | --- | --- | +| `POLICY-ENGINE-29-002-CONTRACT` | DONE (2025-11-23) | Streaming simulation contract published at `docs/modules/policy/contracts/29-002-streaming-simulation.md`; unblocks 29-003..40-002 chain. | | `PREP-EXPORT-CONSOLE-23-001` | DOING (2025-11-20) | Drafted export bundle + scheduler job contract (see `docs/modules/policy/design/export-console-bundle-contract.md`); waiting on DSSE/storage decisions from Console/Scheduler/Authority. | | `PREP-POLICY-AIRGAP-56-001` | DOING (2025-11-20) | Drafted mirror bundle schema for air-gap/ sealed mode (see `docs/modules/policy/design/policy-mirror-bundle-schema.md`); waiting on trust-root and retention policy decisions. | | `PREP-POLICY-ENGINE-30-001` | DOING (2025-11-20) | Drafted overlay projection contract (see `docs/modules/policy/design/policy-overlay-projection.md`); waiting on 29-004 metrics/log schema from Platform/Observability. | diff --git a/docs/modules/policy/contracts/29-002-streaming-simulation.md b/docs/modules/policy/contracts/29-002-streaming-simulation.md new file mode 100644 index 000000000..1f59c096c --- /dev/null +++ b/docs/modules/policy/contracts/29-002-streaming-simulation.md @@ -0,0 +1,114 @@ +# POLICY-ENGINE-29-002 — Streaming Simulation (Path/Scope) Contract + +Status: Final · 2025-11-23 +Owners: Policy Guild · SBOM Service Guild · Findings Ledger Guild +Working directory: `src/Policy/StellaOps.Policy.Engine` + +## Purpose +Define the canonical request/response contract for streaming comparisons between two policy versions with path/scope awareness. Output is explainable per-finding deltas without writes, suitable for Console simulation, Ledger projections, and Advisory AI consumers. + +## Request Schema (JSON) +```json +{ + "schemaVersion": "1.0.0", + "tenant": "acme", // required + "basePolicyRef": "policy://acme/main@sha256:abcd", // required + "candidatePolicyRef": "policy://acme/feature@sha256:ef01", // required + "subject": { + "purl": "pkg:npm/lodash@4.17.21", // or cpe; at least one + "packagePath": "lib/isEqual.js", // optional, POSIX + "osImage": "ghcr.io/acme/app@sha256:..." // optional + }, + "targets": [ + { + "filePath": "src/lib/isEqual.js", // required, POSIX + "digest": "c1ab...", // optional hex sha256 + "treeDigest": "7ff0...", // optional Merkle root + "pathMatch": "prefix", // exact|prefix|glob + "pattern": "src/lib/", // required with pathMatch + "confidence": 0.92, // 0..1 + "depthLimit": 3, // optional int + "evidenceHash": "4f9b...", // optional hex (stable over sorted JSON) + "ingestedAt": "2025-11-20T00:00:00Z", // optional ISO-8601 UTC + "connectorId": "excititor-ghsa" // optional string + } + ], + "options": { + "sort": "path,finding,verdict", // required enum; default shown + "maxFindings": 1000, // optional int + "includeTrace": true, // include rule trace fragments + "deterministic": true // must be true; rejects false + } +} +``` +- `schemaVersion` pinned to semantic version; breaking changes require bump. +- All string comparisons are case-sensitive except `policy://` refs, which normalise to lowercase host/path. +- At least one `target` required. Each target binds evidence to scope (derived from PREP docs `2025-11-20-policy-engine-29-002-prep.md`). + +### Derived Scope Normalisation +For each target: +- `pathMatch` resolution order: `exact` > `prefix` > `glob`; tie-breaker by higher `confidence`, then lexical `filePath`. +- `evidenceHash` = SHA-256 of canonical JSON object with properties sorted ASCII and nulls removed. + +## Response Schema (streamed NDJSON) +Each line is a JSON object with stable ordering: +```json +{ + "tenant": "acme", + "subject": { "purl": "pkg:npm/lodash@4.17.21", "packagePath": "lib/isEqual.js" }, + "target": { + "filePath": "src/lib/isEqual.js", + "pattern": "src/lib/", + "pathMatch": "prefix", + "confidence": 0.92, + "evidenceHash": "4f9b..." + }, + "finding": { + "id": "GHSA-35jh-r3h4-6jhm", + "ruleId": "policy.rules.javascript.unsafe-merge", + "severity": "high", + "verdict": { + "base": "deny", // allow|deny|warn|info|not-applicable + "candidate": "warn", + "delta": "softened" // added|removed|hardened|softened|unchanged + }, + "evidence": { + "locator": { "filePath": "src/lib/isEqual.js", "digest": "c1ab..." }, + "provenance": { "ingestedAt": "2025-11-20T00:00:00Z", "connectorId": "excititor-ghsa" } + } + }, + "trace": [ + { "step": "match", "rule": "unsafe-merge", "path": "src/lib/isEqual.js" }, + { "step": "decision", "effect": "warn" } + ], + "metrics": { + "evalTicks": 128, // deterministic instruction counter + "rulesEvaluated": 12, + "bindings": 3 + } +} +``` +- Lines are sorted by `target.filePath`, then `finding.id`, then `finding.ruleId`. No other ordering allowed. +- Absent optional fields must be omitted (not null). + +## Error Envelope +Errors surface as NDJSON line with `type: "error"`: +```json +{ "type": "error", "code": "POLICY_29_002_SCHEMA", "message": "target[0].pattern required" } +``` +Codes: `POLICY_29_002_SCHEMA`, `POLICY_29_002_UNSUPPORTED_VERSION`, `POLICY_29_002_SCOPE_MISMATCH`, `POLICY_29_002_TOO_MANY_FINDINGS`. + +## Determinism Rules +- No wall-clock, RNG, or network access during evaluation. +- All hashes use SHA-256 over canonical JSON (sorted properties, UTF-8, no insignificant whitespace). +- Stable ordering: request-specified sort or default `path,finding,verdict` defined above. +- Trace fragments trimmed to deterministic steps only (no timestamps or hostnames). + +## Validation & Conformance +- JSON Schema published at `schemas/policy/29-002-streaming-simulation.schema.json` (to be mirrored in Offline Kit); engines must validate requests before execution. +- CI fixtures: `tests/policy/fixtures/29-002-sample-request.json` and `...-response.ndjson` (to be added when engine implements). + +## Compatibility Notes +- Extends PREP docs `2025-11-20-policy-engine-29-002-prep.md` and `2025-11-21-policy-path-scope-29-002-prep.md`; supersedes their draft status. +- Downstream tasks 29-003..40-002 must consume `target` shape above; any change requires bumping `schemaVersion` and updating sprint risks. + diff --git a/docs/modules/scanner/design/surface-env-release.md b/docs/modules/scanner/design/surface-env-release.md new file mode 100644 index 000000000..d3f3814b8 --- /dev/null +++ b/docs/modules/scanner/design/surface-env-release.md @@ -0,0 +1,38 @@ +# Surface.Env Package Release Note + +Status: Published · 2025-11-23 +Owners: Scanner Guild · BuildX Plugin Guild · Ops Guild +Scope: Unblock SURFACE-ENV-03 and BuildX adoption by pinning package version + offline bundle path for `StellaOps.Scanner.Surface.Env`. + +## Version & Build Inputs +- **Package ID:** `StellaOps.Scanner.Surface.Env` +- **Version:** `0.1.0-alpha.20251123` (semantic, date-stamped for sprint 136) +- **Source:** `src/Scanner/__Libraries/StellaOps.Scanner.Surface.Env/StellaOps.Scanner.Surface.Env.csproj` +- **Pack command:** + - `dotnet pack src/Scanner/__Libraries/StellaOps.Scanner.Surface.Env/StellaOps.Scanner.Surface.Env.csproj -c Release -o local-nugets /p:Version=0.1.0-alpha.20251123` +- **Restore sources:** `local-nugets/; dotnet-public; nuget.org` (per `Directory.Build.props`). + +## Offline / Air-Gap Artefacts +- Copy the produced `.nupkg` to `offline/packages/nugets/StellaOps.Scanner.Surface.Env.0.1.0-alpha.20251123.nupkg`. +- Manifest entry: + - `packageId`: `StellaOps.Scanner.Surface.Env` + - `version`: `0.1.0-alpha.20251123` + - `sha256`: `7f79ec14cc52f0880904eccb6fbc8120bd7d316ab8d6390fef054ed11ee4716e` + - `size`: `14080` + - `createdAt`: `2025-11-23T00:00:00Z` +- No external network calls are required after packing; the offline kit consumes the local file. + +## Consumer Guidance +- BuildX plugin (`src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin`) should reference `0.1.0-alpha.20251123` via the curated feed `local-nugets/`. +- Scanner WebService/Worker should use the same version once Surface.Env integration tests pass (SCANNER-ENV-02). +- Surface.Validation and Surface.Secrets depend on the env settings; keep prefix defaults and determinism rules from `design/surface-env.md`. + +## Verification +- Run `dotnet test` for env library once restore is stable; until then, manual pack is acceptable for BuildX smoke tests. +- Validate package contents: + - Contains `StellaOps.Scanner.Surface.Env.dll` and `StellaOps.Scanner.Surface.Env.xml` docs. + - `lib/net10.0/` target only; no native assets. +- Ensure `local-nugets/` feed lists the package with `nuget list -Source local-nugets` before wiring CI. + +## Change Log +- 2025-11-23: Initial release note created to unblock SCANNER-ENV-03 and offline kit wiring; version pinned to `0.1.0-alpha.20251123`. diff --git a/docs/modules/scanner/design/surface-secrets-schema.md b/docs/modules/scanner/design/surface-secrets-schema.md new file mode 100644 index 000000000..694993942 --- /dev/null +++ b/docs/modules/scanner/design/surface-secrets-schema.md @@ -0,0 +1,119 @@ +# Surface.Secrets Schema (Security-approved) + +Status: Final · 2025-11-23 +Owners: Security Guild · Scanner Guild · Zastava Guild +Related tasks: SURFACE-SECRETS-01/02/03/04/05/06, SCANNER-SECRETS-03, ZASTAVA-SECRETS-01/02 + +## Purpose +Canonical schema for declaring and validating secret providers used by Surface consumers (Scanner, Zastava, Scheduler). Supersedes the draft notes in `surface-secrets.md` §4–5 and unblocks validation + integration tasks. + +## Configuration Document +Location: `Surface:Secrets` (appsettings, env, or offline kit manifest). JSON Schema (v2020-12): +```json +{ + "$id": "https://stellaops/policy/surface-secrets.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["provider", "providers"], + "properties": { + "provider": {"type": "string", "enum": ["kubernetes", "file", "inline"], "description": "active provider id"}, + "fallbackProvider": {"type": "string", "enum": ["kubernetes", "file", "inline"], "nullable": true}, + "providers": { + "type": "object", + "properties": { + "kubernetes": { + "type": "object", + "required": ["namespace", "prefix"], + "properties": { + "namespace": {"type": "string", "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$"}, + "prefix": {"type": "string", "default": "surface-"}, + "ttlSeconds": {"type": "integer", "minimum": 30, "default": 600}, + "allowServiceAccountFallback": {"type": "boolean", "default": false} + } + }, + "file": { + "type": "object", + "required": ["root"], + "properties": { + "root": {"type": "string", "pattern": "^/.+"}, + "format": {"type": "string", "enum": ["json", "yaml"], "default": "json"}, + "permissions": {"type": "string", "enum": ["0600"], "default": "0600"} + } + }, + "inline": { + "type": "object", + "properties": { + "payloadBase64": {"type": "string", "contentEncoding": "base64"}, + "enabled": {"type": "boolean", "default": false} + } + } + }, + "additionalProperties": false + }, + "requiredSecrets": { + "type": "array", + "items": { + "type": "object", + "required": ["tenant", "component", "secretType"], + "properties": { + "tenant": {"type": "string"}, + "component": {"type": "string"}, + "secretType": {"type": "string", "enum": ["cas-access", "registry", "attestation", "tls"]}, + "name": {"type": "string"} + } + }, + "uniqueItems": true + } + }, + "additionalProperties": false +} +``` + +### Secret Object Shape +Secrets resolved by providers must conform to one of: +- `cas-access`: `{ "accessKey": "...", "secretKey": "...", "sessionToken": "..." }` +- `registry`: `{ "username": "...", "password": "..." }` or `{ "token": "..." }` +- `attestation`: `{ "privateKeyPem": "...", "keyId": "...", "rekorApiKey": "..." }` +- `tls`: `{ "certPem": "...", "keyPem": "..." }` +Binary values may be base64 but must decode to UTF-8 cleanly; otherwise return base64 string and mark `isBinary: true` in handle metadata. + +## Determinism & Validation +- Deterministic ordering: providers resolved in request order; when multiple secrets share tenant/component/type, choose lexicographically smallest `name` then earliest `ingestedAt` metadata. +- `Surface.Validation` codes: `SURFACE_SECRET_PROVIDER_UNKNOWN`, `SURFACE_SECRET_MISSING`, `SURFACE_SECRET_STALE`, `SURFACE_SECRET_FORMAT_INVALID` (new). +- Validators must reject world-readable files (`permissions != 0600`) and inline payloads when `enabled=false` (air-gapped safety). + +## Offline / Air-Gap Profile +- Offline kit must ship `offline/secrets/manifest.json` matching this schema with hashes for each secret blob (SHA-256 hex) and `createdAt` in UTC. +- Importer scripts map manifest entries to file provider layout: `////.json`. +- No external network calls during validation or resolution. + +## Examples +Minimal Kubernetes config: +```json +{ + "provider": "kubernetes", + "providers": {"kubernetes": {"namespace": "stellaops-runtime", "prefix": "surface-"}}, + "requiredSecrets": [ + {"tenant": "acme", "component": "scanner-worker", "secretType": "cas-access"}, + {"tenant": "acme", "component": "scanner-worker", "secretType": "registry", "name": "primary"} + ] +} +``` + +File provider (offline): +```json +{ + "provider": "file", + "providers": {"file": {"root": "/etc/stellaops/secrets", "format": "json"}}, + "requiredSecrets": [] +} +``` + +## Implementation Notes +- Aligns with design doc `surface-secrets.md`; this file is the authoritative schema for SURFACE-SECRETS-01/02/03. +- `ISurfaceSecretProvider` must emit metadata `{ "provider": "kubernetes", "ageDays": , "isBinary": }` for observability. +- Add JSON Schema to `schemas/surface/surface-secrets.schema.json` (mirrored in Offline Kit) when code lands. + +## Acceptance +- SECURITY sign-off requires adherence to this schema and validation rules. +- SURFACE-SECRETS-01 moves to DONE; SURFACE-SECRETS-02 to TODO (implementation); SURFACE-VAL-01 unblocks because schema is defined. diff --git a/docs/modules/scanner/design/surface-validation.md b/docs/modules/scanner/design/surface-validation.md index 0022c3bad..989304888 100644 --- a/docs/modules/scanner/design/surface-validation.md +++ b/docs/modules/scanner/design/surface-validation.md @@ -1,6 +1,6 @@ # Surface.Validation Design (Epic: SURFACE-SHARING) -> **Status:** Draft v1.0 — aligns with tasks `SURFACE-VAL-01..05`, `LANG-SURFACE-01..03`, `ENTRYTRACE-SURFACE-01..02`, `ZASTAVA-SURFACE-02`, `SCANNER-SECRETS-01..03`. +> **Status:** v1.1 (2025-11-23) — aligned to Surface.Secrets schema (`surface-secrets-schema.md`) and Surface.Env release 0.1.0-alpha.20251123; covers tasks `SURFACE-VAL-01..05`, `LANG-SURFACE-01..03`, `ENTRYTRACE-SURFACE-01..02`, `ZASTAVA-SURFACE-02`, `SCANNER-SECRETS-01..03`. > > **Audience:** Engineers integrating Surface Env/FS/Secrets, QA guild, Security guild. @@ -52,6 +52,7 @@ public sealed record SurfaceValidationIssue( | `SURFACE_ENV_CACHE_DIR_UNWRITABLE` | Error | Cache root not writable or disk full. | | `SURFACE_SECRET_MISSING` | Error | Secret provider cannot locate required secret type. | | `SURFACE_SECRET_STALE` | Warning | Secret older than rotation window. | +| `SURFACE_SECRET_FORMAT_INVALID` | Error | Secret payload fails schema validation per `surface-secrets-schema.md`. | | `SURFACE_FS_ENDPOINT_REACHABILITY` | Error | HEAD request to Surface.FS endpoint failed. | | `SURFACE_FS_BUCKET_MISMATCH` | Error | Provided bucket does not exist / lacks permissions. | | `SURFACE_FEATURE_UNKNOWN` | Warning | Feature flag not recognised. | @@ -71,6 +72,11 @@ services.AddSurfaceValidation(builder => Validators can access DI services (e.g., HttpClient, Authority token provider) through the context. To avoid long-running checks, recommended max validation time is 500ms per validator. +### Secrets schema binding +- Validators must load the secrets configuration into the JSON Schema defined in `surface-secrets-schema.md`; reject when provider/fallback ids are outside the allowed set or when file permissions differ from `0600`. +- When `Surface:Secrets:Provider = file`, ensure each required secret exists at `////.json` with base64 payload matching the secret type contract (see §2 in `surface-secrets-schema.md`). +- Inline provider must be disabled in production (`AllowInline=false`); validation emits `SURFACE_SECRET_FORMAT_INVALID` if enabled without an explicit dev/test flag. + ## 5. Reporting & Observability - Results exposed via `ISurfaceValidationReporter` (default logs structured JSON to `Validation` category). diff --git a/local-nugets/StellaOps.Scanner.Surface.Env.0.1.0-alpha.20251123.nupkg b/local-nugets/StellaOps.Scanner.Surface.Env.0.1.0-alpha.20251123.nupkg new file mode 100644 index 000000000..55ffc094f Binary files /dev/null and b/local-nugets/StellaOps.Scanner.Surface.Env.0.1.0-alpha.20251123.nupkg differ diff --git a/offline/packages/nugets/StellaOps.Scanner.Surface.Env.0.1.0-alpha.20251123.nupkg b/offline/packages/nugets/StellaOps.Scanner.Surface.Env.0.1.0-alpha.20251123.nupkg new file mode 100644 index 000000000..55ffc094f Binary files /dev/null and b/offline/packages/nugets/StellaOps.Scanner.Surface.Env.0.1.0-alpha.20251123.nupkg differ diff --git a/ops/deployment/export/helm-overlays.md b/ops/deployment/export/helm-overlays.md new file mode 100644 index 000000000..7eaeff90d --- /dev/null +++ b/ops/deployment/export/helm-overlays.md @@ -0,0 +1,35 @@ +# Export Center Helm Overlays (DEPLOY-EXPORT-35-001) + +## Values files (download-only) +- `deploy/helm/stellaops/values-export.yaml` (add) with: + - `exportcenter:` + - `image.repository`: `registry.stella-ops.org/export-center` + - `image.tag`: set via pipeline + - `objectStorage.endpoint`: `http://minio:9000` + - `objectStorage.bucket`: `export-prod` + - `objectStorage.accessKeySecret`: `exportcenter-minio` + - `objectStorage.secretKeySecret`: `exportcenter-minio` + - `signing.kmsKey`: `exportcenter-kms` + - `signing.kmsRegion`: `us-east-1` + - `dsse.enabled`: true + +## Secrets +- KMS signing: create secret `exportcenter-kms` with JSON key material (KMS provider specific). Example: `ops/deployment/export/secrets-example.yaml`. +- MinIO creds: `exportcenter-minio` with `accesskey`, `secretkey` keys (see example manifest). + +## Rollout +- `helm upgrade --install export-center deploy/helm/stellaops -f deploy/helm/stellaops/values-export.yaml --set image.tag=$TAG` +- Pre-flight: `helm template ...` and `helm lint`. +- Post: verify readiness `kubectl rollout status deploy/export-center` and run `curl /healthz`. + +## Rollback +- `helm rollback export-center `; ensure previous tag exists. + +## Required artefacts +- Signed images + provenance (from release pipeline). +- SBOM attached via registry (cosign attestations acceptable). + +## Acceptance +- Overlay renders without missing values. +- Secrets documented and referenced in template. +- Rollout/rollback steps documented. diff --git a/ops/deployment/export/secrets-example.yaml b/ops/deployment/export/secrets-example.yaml new file mode 100644 index 000000000..35cced13b --- /dev/null +++ b/ops/deployment/export/secrets-example.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Secret +metadata: + name: exportcenter-minio +stringData: + accesskey: REPLACE_ME + secretkey: REPLACE_ME +--- +apiVersion: v1 +kind: Secret +metadata: + name: exportcenter-kms +stringData: + key.json: | + {"kmsProvider":"awskms","keyId":"arn:aws:kms:...","region":"us-east-1"} diff --git a/ops/deployment/notify/helm-overlays.md b/ops/deployment/notify/helm-overlays.md new file mode 100644 index 000000000..65a7e988d --- /dev/null +++ b/ops/deployment/notify/helm-overlays.md @@ -0,0 +1,28 @@ +# Notifier Helm Overlays (DEPLOY-NOTIFY-38-001) + +## Values file +- `deploy/helm/stellaops/values-notify.yaml` (added) with: + - `notify:` + - `image.repository`: `registry.stella-ops.org/notify` + - `image.tag`: set by pipeline + - `smtp.host`, `smtp.port`, `smtp.usernameSecret`, `smtp.passwordSecret` + - `webhook.allowedHosts`: list + - `chat.webhookSecret`: secret name for chat tokens + - `tls.secretName`: optional ingress cert + +## Secrets +- SMTP creds secret `notify-smtp` with keys `username`, `password` (see `ops/deployment/notify/secrets-example.yaml`). +- Chat/webhook secret `notify-chat` with key `token` (see example manifest). + +## Rollout +- `helm upgrade --install notify deploy/helm/stellaops -f deploy/helm/stellaops/values-notify.yaml --set image.tag=$TAG` +- Pre-flight: `helm lint`, `helm template`. +- Post: `kubectl rollout status deploy/notify` and `curl /healthz`. + +## Rollback +- `helm rollback notify `; confirm previous image tag exists. + +## Acceptance +- Overlay renders without missing values. +- Secrets documented and referenced. +- Rollout/rollback steps documented. diff --git a/ops/deployment/notify/secrets-example.yaml b/ops/deployment/notify/secrets-example.yaml new file mode 100644 index 000000000..be44cd650 --- /dev/null +++ b/ops/deployment/notify/secrets-example.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Secret +metadata: + name: notify-smtp +stringData: + username: REPLACE_ME + password: REPLACE_ME +--- +apiVersion: v1 +kind: Secret +metadata: + name: notify-chat +stringData: + token: REPLACE_ME diff --git a/ops/devops/aoc/aoc-ci.md b/ops/devops/aoc/aoc-ci.md new file mode 100644 index 000000000..6b27dc9ff --- /dev/null +++ b/ops/devops/aoc/aoc-ci.md @@ -0,0 +1,25 @@ +# AOC Analyzer CI Contract (DEVOPS-AOC-19-001) + +## Scope +Integrate AOC Roslyn analyzer and guard tests into CI to block banned writes in ingestion projects. + +## Steps +1) Restore & build analyzers + - `dotnet restore src/Aoc/__Analyzers/StellaOps.Aoc.Analyzers/StellaOps.Aoc.Analyzers.csproj` + - `dotnet build src/Aoc/__Analyzers/StellaOps.Aoc.Analyzers/StellaOps.Aoc.Analyzers.csproj -c Release` +2) Run analyzer on ingestion projects (Authority/Concelier/Excititor ingest paths) + - `dotnet build src/Concelier/StellaOps.Concelier.Ingestion/StellaOps.Concelier.Ingestion.csproj -c Release /p:RunAnalyzers=true /p:TreatWarningsAsErrors=true` + - `dotnet build src/Authority/StellaOps.Authority.Ingestion/StellaOps.Authority.Ingestion.csproj -c Release /p:RunAnalyzers=true /p:TreatWarningsAsErrors=true` + - `dotnet build src/Excititor/StellaOps.Excititor.Ingestion/StellaOps.Excititor.Ingestion.csproj -c Release /p:RunAnalyzers=true /p:TreatWarningsAsErrors=true` +3) Guard tests + - `dotnet test src/Aoc/__Tests/StellaOps.Aoc.Analyzers.Tests/StellaOps.Aoc.Analyzers.Tests.csproj -c Release` +4) Artefacts + - Upload `.artifacts/aoc-analyzer.log` and test TRX. + +## Determinism/Offline +- Use local feeds (`local-nugets/`); no external fetches post-restore. +- Build with `/p:ContinuousIntegrationBuild=true`. + +## Acceptance +- CI fails on any analyzer warning in ingestion projects. +- Tests pass; artefacts uploaded. diff --git a/ops/devops/aoc/aoc-verify-stage.md b/ops/devops/aoc/aoc-verify-stage.md new file mode 100644 index 000000000..95be3880f --- /dev/null +++ b/ops/devops/aoc/aoc-verify-stage.md @@ -0,0 +1,22 @@ +# AOC Verify Stage (DEVOPS-AOC-19-002) + +## Purpose +Add CI stage to run `stella aoc verify --since ` against seeded Mongo snapshots for Concelier + Excititor, publishing violation reports. + +## Inputs +- `STAGING_MONGO_URI` (read-only snapshot). +- Optional `AOC_VERIFY_SINCE` (defaults to `HEAD~1`). + +## Steps +1) Seed snapshot (if needed) + - Restore snapshot into local Mongo or point to read-only staging snapshot. +2) Run verify + - `dotnet run --project src/Aoc/StellaOps.Aoc.Cli -- verify --since ${AOC_VERIFY_SINCE:-HEAD~1} --mongo $STAGING_MONGO_URI --output .artifacts/aoc-verify.json` +3) Fail on violations + - Parse `.artifacts/aoc-verify.json`; if `violations > 0`, fail with summary. +4) Publish artifacts + - Upload `.artifacts/aoc-verify.json` and `.artifacts/aoc-verify.ndjson` (per-violation). + +## Acceptance +- Stage fails when violations exist; passes clean otherwise. +- Artifacts attached for auditing. diff --git a/ops/devops/export/trivy-smoke.sh b/ops/devops/export/trivy-smoke.sh new file mode 100644 index 000000000..2c0225e61 --- /dev/null +++ b/ops/devops/export/trivy-smoke.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail +# Smoke tests for Trivy compatibility and OCI distribution for Export Center. +ROOT=${ROOT:-$(cd "$(dirname "$0")/../.." && pwd)} +ARTifacts=${ARTifacts:-$ROOT/out/export-smoke} +mkdir -p "$ARTifacts" + +# 1) Trivy DB import compatibility +TRIVY_VERSION="0.52.2" +TRIVY_BIN="$ARTifacts/trivy" +if [[ ! -x "$TRIVY_BIN" ]]; then + curl -fsSL "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" -o "$ARTifacts/trivy.tgz" + tar -xzf "$ARTifacts/trivy.tgz" -C "$ARTifacts" trivy +fi +"$TRIVY_BIN" module db import --help > "$ARTifacts/trivy-import-help.txt" + +# 2) OCI distribution check (local registry) +REGISTRY_PORT=${REGISTRY_PORT:-5005} +REGISTRY_DIR="$ARTifacts/registry" +mkdir -p "$REGISTRY_DIR" +podman run --rm -d -p "${REGISTRY_PORT}:5000" --name export-registry -v "$REGISTRY_DIR":/var/lib/registry registry:2 +trap 'podman rm -f export-registry >/dev/null 2>&1 || true' EXIT +sleep 2 + +echo '{"schemaVersion":2,"manifests":[]}' > "$ARTifacts/empty-index.json" +DIGEST=$(sha256sum "$ARTifacts/empty-index.json" | awk '{print $1}') +mkdir -p "$ARTifacts/blobs/sha256" +cp "$ARTifacts/empty-index.json" "$ARTifacts/blobs/sha256/$DIGEST" + +# Push blob and manifest via curl +cat > "$ARTifacts/manifest.json" < "$ARTifacts/result.txt" diff --git a/ops/devops/lnm/backfill-plan.md b/ops/devops/lnm/backfill-plan.md new file mode 100644 index 000000000..76af40c2c --- /dev/null +++ b/ops/devops/lnm/backfill-plan.md @@ -0,0 +1,32 @@ +# LNM Backfill Plan (DEVOPS-LNM-22-001) + +## Goal +Run staging backfill for advisory observations/linksets, validate counts/conflicts, and document rollout steps for production. + +## Prereqs +- Concelier API CCLN0102 available (advisory/linkset endpoints stable). +- Staging Mongo snapshot taken (pre-backfill) and stored at `s3://staging-backups/concelier-pre-lnmbf.gz`. +- NATS/Redis staging brokers reachable. + +## Steps +1) Seed snapshot + - Restore staging Mongo from pre-backfill snapshot. +2) Run backfill job + - `dotnet run --project src/Concelier/StellaOps.Concelier.Backfill -- --mode=observations --batch-size=500 --max-conflicts=0` + - `dotnet run --project src/Concelier/StellaOps.Concelier.Backfill -- --mode=linksets --batch-size=500 --max-conflicts=0` +3) Validate counts + - Compare `advisory_observations_total` and `linksets_total` vs expected inventory; export to `.artifacts/lnm-counts.json`. + - Check conflict log `.artifacts/lnm-conflicts.ndjson` (must be empty). +4) Events/NATS smoke + - Ensure `concelier.lnm.backfill.completed` emitted; verify Redis/NATS queues drained. +5) Roll-forward checklist + - Promote batch size to 2000 for prod, keep `--max-conflicts=0`. + - Schedule maintenance window, ensure snapshot available for rollback. + +## Outputs +- `.artifacts/lnm-counts.json` +- `.artifacts/lnm-conflicts.ndjson` (empty) +- Log of job runtime + throughput. + +## Acceptance +- Zero conflicts; counts match expected; events emitted; rollback plan documented. diff --git a/ops/devops/lnm/backfill-validation.sh b/ops/devops/lnm/backfill-validation.sh new file mode 100644 index 000000000..d7a077ca8 --- /dev/null +++ b/ops/devops/lnm/backfill-validation.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT=${ROOT:-$(cd "$(dirname "$0")/../.." && pwd)} +ARTifacts=${ARTifacts:-$ROOT/.artifacts} +COUNTS=$ARTifacts/lnm-counts.json +CONFLICTS=$ARTifacts/lnm-conflicts.ndjson +mkdir -p "$ARTifacts" + +mongoexport --uri "${STAGING_MONGO_URI:?set STAGING_MONGO_URI}" --collection advisoryObservations --db concelier --type=json --query '{}' --out "$ARTifacts/obs.json" >/dev/null +mongoexport --uri "${STAGING_MONGO_URI:?set STAGING_MONGO_URI}" --collection linksets --db concelier --type=json --query '{}' --out "$ARTifacts/linksets.json" >/dev/null + +OBS=$(jq length "$ARTifacts/obs.json") +LNK=$(jq length "$ARTifacts/linksets.json") + +cat > "$COUNTS" </dev/null +REQUIRED=("advisory_observations_total" "linksets_total" "ingest_api_latency_seconds_bucket" "lnm_backfill_processed_total") +for metric in "${REQUIRED[@]}"; do + if ! grep -q "$metric" "$DASHBOARD"; then + echo "::error::metric $metric missing from dashboard"; exit 1 + fi +done +echo "dashboard metrics present" diff --git a/ops/devops/lnm/metrics-dashboard.json b/ops/devops/lnm/metrics-dashboard.json new file mode 100644 index 000000000..d62fa17b6 --- /dev/null +++ b/ops/devops/lnm/metrics-dashboard.json @@ -0,0 +1,9 @@ +{ + "title": "LNM Backfill Metrics", + "panels": [ + {"type": "stat", "title": "Observations", "targets": [{"expr": "advisory_observations_total"}]}, + {"type": "stat", "title": "Linksets", "targets": [{"expr": "linksets_total"}]}, + {"type": "graph", "title": "Ingest→API latency p95", "targets": [{"expr": "histogram_quantile(0.95, rate(ingest_api_latency_seconds_bucket[5m]))"}]}, + {"type": "graph", "title": "Backfill throughput", "targets": [{"expr": "rate(lnm_backfill_processed_total[5m])"}]} + ] +} diff --git a/ops/devops/lnm/vex-backfill-plan.md b/ops/devops/lnm/vex-backfill-plan.md new file mode 100644 index 000000000..3ddcb564b --- /dev/null +++ b/ops/devops/lnm/vex-backfill-plan.md @@ -0,0 +1,20 @@ +# VEX Backfill Plan (DEVOPS-LNM-22-002) + +## Goal +Run VEX observation/linkset backfill with monitoring, ensure events flow via NATS/Redis, and capture run artifacts. + +## Steps +1) Pre-checks + - Confirm DEVOPS-LNM-22-001 counts baseline (`.artifacts/lnm-counts.json`). + - Ensure `STAGING_MONGO_URI`, `NATS_URL`, `REDIS_URL` available (read-only or test brokers). +2) Run VEX backfill + - `dotnet run --project src/Concelier/StellaOps.Concelier.Backfill -- --mode=vex --batch-size=500 --max-conflicts=0 --mongo $STAGING_MONGO_URI --nats $NATS_URL --redis $REDIS_URL` +3) Metrics capture + - Export per-run metrics to `.artifacts/vex-backfill-metrics.json` (duration, processed, conflicts, events emitted). +4) Event verification + - Subscribe to `concelier.vex.backfill.completed` and `concelier.linksets.vex.upserted`; ensure queues drained. +5) Roll-forward checklist + - Increase batch size to 2000 for prod; keep conflicts = 0; schedule maintenance window. + +## Acceptance +- Zero conflicts; events observed; metrics file present; rollback plan documented. diff --git a/out/mirror/thin/mirror-thin-v1.manifest.dsse.json b/out/mirror/thin/mirror-thin-v1.manifest.dsse.json index 4ab1b3533..62ebdf1ef 100644 --- a/out/mirror/thin/mirror-thin-v1.manifest.dsse.json +++ b/out/mirror/thin/mirror-thin-v1.manifest.dsse.json @@ -1,10 +1,10 @@ { - "payload": "ewogICJjcmVhdGVkIjogIjIwMjUtMTEtMjNUMDA6MDA6MDBaIiwKICAiaW5kZXhlcyI6IFsKICAgIHsKICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6YjY0YzdlNWQ0NDA4YTEwMDMxMWVjOGZhYmM3NmI5ZTUyNTE2NWUyMWRmZmMzZjQ2NDFhZjc5YjlhYTQ0MzNjOSIsCiAgICAgICJuYW1lIjogIm9ic2VydmF0aW9ucy5pbmRleCIKICAgIH0KICBdLAogICJsYXllcnMiOiBbCiAgICB7CiAgICAgICJkaWdlc3QiOiAic2hhMjU2OmZkM2NlNTA0OTdjYmQyMDNkZjIyY2QyZmQxNDY0NmIxYWFjODU4ODRlZDE2MzIxNWE3OWM2MjA3MzAxMjQ1ZDYiLAogICAgICAicGF0aCI6ICJsYXllcnMvb2JzZXJ2YXRpb25zLm5kanNvbiIsCiAgICAgICJzaXplIjogMzEwCiAgICB9CiAgXSwKICAidmVyc2lvbiI6ICIxLjAuMCIKfQo", + "payload": "ewogICJjcmVhdGVkIjogIjIwMjUtMTEtMjNUMDA6MDA6MDBaIiwKICAiaW5kZXhlcyI6IFsKICAgIHsKICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6YjY0YzdlNWQ0NDA4YTEwMDMxMWVjOGZhYmM3NmI5ZTUyNTE2NWUyMWRmZmMzZjQ2NDFhZjc5YjlhYTQ0MzNjOSIsCiAgICAgICJuYW1lIjogIm9ic2VydmF0aW9ucy5pbmRleCIKICAgIH0KICBdLAogICJsYXllcnMiOiBbCiAgICB7CiAgICAgICJkaWdlc3QiOiAic2hhMjU2OmZkM2NlNTA0OTdjYmQyMDNkZjIyY2QyZmQxNDY0NmIxYWFjODU4ODRlZDE2MzIxNWE3OWM2MjA3MzAxMjQ1ZDYiLAogICAgICAicGF0aCI6ICJsYXllcnMvb2JzZXJ2YXRpb25zLm5kanNvbiIsCiAgICAgICJzaXplIjogMzEwCiAgICB9LAogICAgewogICAgICAiZGlnZXN0IjogInNoYTI1NjpjMjdhMGZiMGRmYThhOTU1OGFhYWJmODAxMTA0MGFiY2Q0MTcwY2Y2MmUzNmQxNmI1YjE3NjczNjhmNzgyOGZmIiwKICAgICAgInBhdGgiOiAibGF5ZXJzL3RpbWUtYW5jaG9yLmpzb24iLAogICAgICAic2l6ZSI6IDMyMgogICAgfQogIF0sCiAgInZlcnNpb24iOiAiMS4wLjAiCn0K", "payloadType": "application/vnd.stellaops.mirror.manifest+json", "signatures": [ { - "keyid": "bfac74336bb5d0c8a8c8fe9580fd234ad9f136d0baa6562604a4b49d78282731", - "sig": "zVTaqWzJPPQtD_-8J3AsfwaG4nbS9I7XQXa5aZyIXLaIi_t1BxI_5r96klKfUAB8V-kWvkvjCg3pjmtoKJtzCQ" + "keyid": "db9928babf3aeb817ccdcd0f6a6688f8395b00d0e42966e32e706931b5301fc8", + "sig": "EC7tbq5zlHqUfidvkT-Q1yfmiTJs9KUdpnvs9jCBJXsxzIyB1hzfdh-7FNPi3pFSrzV6cDh47cWvWmMR_ypgDw" } ] } diff --git a/out/mirror/thin/oci/blobs/sha256/1ef17d14c09e74703b88753d6c561d8c8a8809fe8e05972257adadfb91b71723 b/out/mirror/thin/oci/blobs/sha256/1ef17d14c09e74703b88753d6c561d8c8a8809fe8e05972257adadfb91b71723 new file mode 100644 index 000000000..843eb9db3 Binary files /dev/null and b/out/mirror/thin/oci/blobs/sha256/1ef17d14c09e74703b88753d6c561d8c8a8809fe8e05972257adadfb91b71723 differ diff --git a/out/mirror/thin/oci/index.json b/out/mirror/thin/oci/index.json index aa9100b44..1f6eec31f 100644 --- a/out/mirror/thin/oci/index.json +++ b/out/mirror/thin/oci/index.json @@ -3,7 +3,7 @@ "manifests": [ { "mediaType": "application/vnd.oci.image.manifest.v1+json", - "digest": "sha256:f61bfb0206807e1ef79325f34435ae3fd32d90284e82abf649fb2b0c0480adc1", + "digest": "sha256:f6bd80fe9d346e7306c69832e29180346454005a0751c77ae2ebb7332be94642", "size": 485, "annotations": {"org.opencontainers.image.ref.name": "mirror-thin-v1"} } diff --git a/out/mirror/thin/oci/manifest.json b/out/mirror/thin/oci/manifest.json index de50ed44e..f31df21da 100644 --- a/out/mirror/thin/oci/manifest.json +++ b/out/mirror/thin/oci/manifest.json @@ -8,8 +8,8 @@ "layers": [ { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", - "size": 613, - "digest": "sha256:210dc49e8d3e25509298770a94da277aa2c9d4c387d3c24505a61fe1d7695a49", + "size": 830, + "digest": "sha256:1ef17d14c09e74703b88753d6c561d8c8a8809fe8e05972257adadfb91b71723", "annotations": {"org.stellaops.bundle.type": "mirror-thin-v1"} } ] diff --git a/out/mirror/thin/tuf/keys/ci-ed25519.pem b/out/mirror/thin/tuf/keys/ci-ed25519.pem new file mode 100644 index 000000000..be6de4f92 --- /dev/null +++ b/out/mirror/thin/tuf/keys/ci-ed25519.pem @@ -0,0 +1,3 @@ +-----BEGIN PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEIDjozCEWJUQTumqgh2FfWqPZziPnGZK4o8VQM8FbFBHG +-----END PRIVATE KEY----- diff --git a/out/mirror/thin/tuf/keys/ci-ed25519.pub b/out/mirror/thin/tuf/keys/ci-ed25519.pub new file mode 100644 index 000000000..125d721f4 --- /dev/null +++ b/out/mirror/thin/tuf/keys/ci-ed25519.pub @@ -0,0 +1,3 @@ +-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEAhjL0kS93VsBbWqkuPGvu8Hsfe0iSe2E4/gc/TwTNM00= +-----END PUBLIC KEY----- diff --git a/out/mirror/thin/tuf/root.json b/out/mirror/thin/tuf/root.json index 3c3329a29..66b9ee728 100644 --- a/out/mirror/thin/tuf/root.json +++ b/out/mirror/thin/tuf/root.json @@ -22,8 +22,8 @@ }, "signatures": [ { - "keyid": "bfac74336bb5d0c8a8c8fe9580fd234ad9f136d0baa6562604a4b49d78282731", - "sig": "oisYau2KLEHT8luXEFfpXqBYiFHNb4271MwhuptukT69nNijwq-F4_acb_2uP-o7xGOx5pSTVvB8n0DzXvSACQ" + "keyid": "db9928babf3aeb817ccdcd0f6a6688f8395b00d0e42966e32e706931b5301fc8", + "sig": "ZUXDqV5hn0cuZlOOEUZdpD474mc0bkJu4-LyBPNYwU3YkZufT2eXKM-QHksF4JoXgywbY9QD8qhnsEh05xoKBg" } ], "spec_version": "1.0.31", diff --git a/out/mirror/thin/tuf/snapshot.json b/out/mirror/thin/tuf/snapshot.json index f207987a8..c057a4b1c 100644 --- a/out/mirror/thin/tuf/snapshot.json +++ b/out/mirror/thin/tuf/snapshot.json @@ -4,8 +4,8 @@ "meta": {}, "signatures": [ { - "keyid": "bfac74336bb5d0c8a8c8fe9580fd234ad9f136d0baa6562604a4b49d78282731", - "sig": "LF2J66AaYOqwCmTaitnxY2IdtRs6jEHpARV04SRSEUU_WAxprDO1DlcvQn6KcM7IwitOCzYPKVDEZGGhlQs5CA" + "keyid": "db9928babf3aeb817ccdcd0f6a6688f8395b00d0e42966e32e706931b5301fc8", + "sig": "Z2FtwGRtVhQNvNZUxceUb3Ygj5KNqJGTOFIq8CxltBvMfmaAavWmMST0shir7p-7LI3-kBUMdPOKYlGxFip3AQ" } ], "spec_version": "1.0.31", diff --git a/out/mirror/thin/tuf/targets.json b/out/mirror/thin/tuf/targets.json index 859cdaf32..00f61c5f7 100644 --- a/out/mirror/thin/tuf/targets.json +++ b/out/mirror/thin/tuf/targets.json @@ -3,8 +3,8 @@ "expires": "2026-01-01T00:00:00Z", "signatures": [ { - "keyid": "bfac74336bb5d0c8a8c8fe9580fd234ad9f136d0baa6562604a4b49d78282731", - "sig": "RiNTtMhWHmfPJXhVcTq_wvlqrmuYBQlSbc3El0coCvDcbH8bGpyS79igbarj0DnSrVgL48qj3Q33UFEgiY-FAg" + "keyid": "db9928babf3aeb817ccdcd0f6a6688f8395b00d0e42966e32e706931b5301fc8", + "sig": "SIKtu5qz3FYNQxittPQwwWUzQLRg9D6KpO3OKpxtZzrbD2S5corjRZg-JNymPzFoEbrm8i5b_p7sh6H44At-CQ" } ], "spec_version": "1.0.31", diff --git a/out/mirror/thin/tuf/timestamp.json b/out/mirror/thin/tuf/timestamp.json index b55893146..a9017014f 100644 --- a/out/mirror/thin/tuf/timestamp.json +++ b/out/mirror/thin/tuf/timestamp.json @@ -4,8 +4,8 @@ "meta": {}, "signatures": [ { - "keyid": "bfac74336bb5d0c8a8c8fe9580fd234ad9f136d0baa6562604a4b49d78282731", - "sig": "NdOzauVxBqvWIirilXY_SDcvgnx1_LLwUXE-G268eob7RJ_HUSnc_SAt0iGLYDcjBUf3tJL1nz025YWzVkmDCw" + "keyid": "db9928babf3aeb817ccdcd0f6a6688f8395b00d0e42966e32e706931b5301fc8", + "sig": "C_4pXTUzKaVEZ0Dwtn2FlXxOsxcht8nF_vdWwVOMsYqwrqYriZgd4x_r2lq_RnI5QYxagEHGnEjD-6ztEeRMCg" } ], "spec_version": "1.0.31", diff --git a/scripts/mirror/ci-sign.sh b/scripts/mirror/ci-sign.sh index ebdaefdf7..e0e33bae5 100644 --- a/scripts/mirror/ci-sign.sh +++ b/scripts/mirror/ci-sign.sh @@ -2,7 +2,9 @@ set -euo pipefail # Allow CI to fall back to a deterministic test key when MIRROR_SIGN_KEY_B64 is unset, # but forbid this on release/tag builds when REQUIRE_PROD_SIGNING=1. -DEFAULT_TEST_KEY_B64="LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1DNENBUUF3QlFZREsyVndCQ0lFSUhLbjhWMjJ5ZEpwbkZTY3k5VlNsdTczNXZBQ1NFdFFIWlBRR3pSNzcyUGcKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo=" +# Throwaway dev key (Ed25519) generated 2025-11-23; matches the value documented in +# docs/modules/mirror/signing-runbook.md. Safe for non-production smoke only. +DEFAULT_TEST_KEY_B64="LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1DNENBUUF3QlFZREsyVndCQ0lFSURqb3pDRVdKVVFUdW1xZ2gyRmZXcVBaemlQbkdaSzRvOFZRTThGYkZCSEcKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo=" if [[ -z "${MIRROR_SIGN_KEY_B64:-}" ]]; then if [[ "${REQUIRE_PROD_SIGNING:-0}" == "1" ]]; then echo "[error] MIRROR_SIGN_KEY_B64 is required for production signing; refusing to use test key." >&2 @@ -17,6 +19,8 @@ mkdir -p "$KEYDIR" KEYFILE="$KEYDIR/ci-ed25519.pem" printf "%s" "$MIRROR_SIGN_KEY_B64" | base64 -d > "$KEYFILE" chmod 600 "$KEYFILE" +# Export public key for TUF keyid calculation +openssl pkey -in "$KEYFILE" -pubout -out "$KEYDIR/ci-ed25519.pub" >/dev/null 2>&1 STAGE=${STAGE:-$ROOT/out/mirror/thin/stage-v1} CREATED=${CREATED:-$(date -u +%Y-%m-%dT%H:%M:%SZ)} SIGN_KEY="$KEYFILE" STAGE="$STAGE" CREATED="$CREATED" "$ROOT/src/Mirror/StellaOps.Mirror.Creator/make-thin-v1.sh" diff --git a/src/Excititor/StellaOps.Excititor.WebService/Program.cs b/src/Excititor/StellaOps.Excititor.WebService/Program.cs index d9a973a74..e1b311ffa 100644 --- a/src/Excititor/StellaOps.Excititor.WebService/Program.cs +++ b/src/Excititor/StellaOps.Excititor.WebService/Program.cs @@ -32,7 +32,6 @@ using StellaOps.Excititor.WebService.Extensions; using StellaOps.Excititor.WebService.Options; using StellaOps.Excititor.WebService.Services; using StellaOps.Excititor.Core.Aoc; -using StellaOps.Excititor.WebService.Contracts; using StellaOps.Excititor.WebService.Telemetry; using MongoDB.Driver; using MongoDB.Bson; @@ -170,14 +169,14 @@ app.MapPost("/airgap/v1/vex/import", async ( if (!trustService.Validate(request, out var trustCode, out var trustMessage)) { - return Results.StatusCode(StatusCodes.Status403Forbidden, new + return Results.Json(new { error = new { code = trustCode, message = trustMessage } - }); + }, statusCode: StatusCodes.Status403Forbidden); } var record = new AirgapImportRecord @@ -344,13 +343,26 @@ app.MapGet("/console/vex", async ( } var query = context.Request.Query; - var purls = query["purl"].Where(static v => !string.IsNullOrWhiteSpace(v)).Select(static v => v.Trim()).ToArray(); - var advisories = query["advisoryId"].Where(static v => !string.IsNullOrWhiteSpace(v)).Select(static v => v.Trim()).ToArray(); + static string[] NormalizeValues(StringValues values) => + values.Where(static v => !string.IsNullOrWhiteSpace(v)) + .Select(static v => v!.Trim()) + .ToArray(); + + var purls = query.TryGetValue("purl", out var purlValues) + ? NormalizeValues(purlValues) + : Array.Empty(); + var advisories = query.TryGetValue("advisoryId", out var advisoryValues) + ? NormalizeValues(advisoryValues) + : Array.Empty(); var statuses = new List(); if (query.TryGetValue("status", out var statusValues)) { foreach (var statusValue in statusValues) { + if (string.IsNullOrWhiteSpace(statusValue)) + { + continue; + } if (Enum.TryParse(statusValue, ignoreCase: true, out var parsed)) { statuses.Add(parsed); @@ -377,17 +389,17 @@ app.MapGet("/console/vex", async ( } telemetry.CacheMisses.Add(1); - var options = new VexObservationQueryOptions( - tenant, - observationIds: null, - vulnerabilityIds: advisories, - productKeys: null, - purls: purls, - cpes: null, - providerIds: null, - statuses: statuses, - cursor: cursor, - limit: limit); +var options = new VexObservationQueryOptions( + tenant, + observationIds: null, + vulnerabilityIds: advisories, + productKeys: null, + purls: purls, + cpes: null, + providerIds: null, + statuses: statuses, + limit: limit, + cursor: cursor); VexObservationQueryResult result; try @@ -399,22 +411,24 @@ app.MapGet("/console/vex", async ( return Results.BadRequest(ex.Message); } - var statements = result.Observations - .SelectMany(obs => obs.Statements.Select(stmt => new VexConsoleStatementDto( - AdvisoryId: stmt.VulnerabilityId, - ProductKey: stmt.ProductKey, - Purl: stmt.Purl ?? obs.Linkset.Purls.FirstOrDefault(), - Status: stmt.Status.ToString().ToLowerInvariant(), - Justification: stmt.Justification?.ToString(), - ProviderId: obs.ProviderId, - ObservationId: obs.ObservationId, - CreatedAtUtc: obs.CreatedAt, - Attributes: obs.Attributes))) - .ToList(); +var statements = result.Observations + .SelectMany(obs => obs.Statements.Select(stmt => new VexConsoleStatementDto( + AdvisoryId: stmt.VulnerabilityId, + ProductKey: stmt.ProductKey, + Purl: stmt.Purl + ?? (obs.Linkset is { } linkset ? linkset.Purls.FirstOrDefault() : null) + ?? string.Empty, + Status: stmt.Status.ToString().ToLowerInvariant(), + Justification: stmt.Justification?.ToString(), + ProviderId: obs.ProviderId, + ObservationId: obs.ObservationId, + CreatedAtUtc: obs.CreatedAt, + Attributes: obs.Attributes ?? ImmutableDictionary.Empty))) + .ToList(); - var statusCounts = result.Observations - .GroupBy(o => o.Status.ToString().ToLowerInvariant()) - .ToDictionary(g => g.Key, g => g.Count(), StringComparer.OrdinalIgnoreCase); +var statusCounts = statements + .GroupBy(o => o.Status) + .ToDictionary(g => g.Key, g => g.Count(), StringComparer.OrdinalIgnoreCase); var response = new VexConsolePage( Items: statements, @@ -455,12 +469,10 @@ app.MapPost("/internal/graph/linkouts", async ( return Results.BadRequest("purls are required (1-500)."); } - var options = new VexObservationQueryOptions( - request.Tenant.Trim(), - purls: normalizedPurls, - includeJustifications: request.IncludeJustifications, - includeProvenance: request.IncludeProvenance, - limit: 200); +var options = new VexObservationQueryOptions( + request.Tenant.Trim(), + purls: normalizedPurls, + limit: 200); VexObservationQueryResult result; try @@ -495,31 +507,18 @@ app.MapPost("/internal/graph/linkouts", async ( Status: stmt.Status.ToString().ToLowerInvariant(), Justification: request.IncludeJustifications ? stmt.Justification?.ToString() : null, ModifiedAt: obs.CreatedAt, - EvidenceHash: obs.Linkset.ReferenceHash, + EvidenceHash: string.Empty, ConnectorId: obs.ProviderId, - DsseEnvelopeHash: request.IncludeProvenance ? obs.Linkset.ReferenceHash : null))) + DsseEnvelopeHash: request.IncludeProvenance ? string.Empty : null))) .OrderBy(a => a.AdvisoryId, StringComparer.Ordinal) .ThenBy(a => a.Source, StringComparer.Ordinal) .Take(200) .ToList(); - var conflicts = obsForPurl - .Where(obs => obs.Statements.Any(s => s.Status == VexClaimStatus.Conflict)) - .SelectMany(obs => obs.Statements - .Where(s => s.Status == VexClaimStatus.Conflict) - .Select(stmt => new GraphLinkoutConflict( - Source: obs.ProviderId, - Status: stmt.Status.ToString().ToLowerInvariant(), - Justification: request.IncludeJustifications ? stmt.Justification?.ToString() : null, - ObservedAt: obs.CreatedAt, - EvidenceHash: obs.Linkset.ReferenceHash))) - .OrderBy(c => c.Source, StringComparer.Ordinal) - .ToList(); - items.Add(new GraphLinkoutItem( Purl: inputPurl, Advisories: advisories, - Conflicts: conflicts, + Conflicts: Array.Empty(), Truncated: advisories.Count >= 200, NextCursor: advisories.Count >= 200 ? $"{advisories[^1].AdvisoryId}:{advisories[^1].Source}" : null)); } diff --git a/src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/AirgapImportEndpointTests.cs b/src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/AirgapImportEndpointTests.cs index 2a23c6b1a..3d189b5bc 100644 --- a/src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/AirgapImportEndpointTests.cs +++ b/src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/AirgapImportEndpointTests.cs @@ -13,10 +13,10 @@ public class AirgapImportEndpointTests var request = new AirgapImportRequest { BundleId = "bundle-123", - MirrorGeneration = "gen-1", + MirrorGeneration = "1", SignedAt = DateTimeOffset.UtcNow, Publisher = "mirror-test", - PayloadHash = "sha256:abc" + PayloadHash = "sha256:" + new string('a', 64) }; var errors = validator.Validate(request, DateTimeOffset.UtcNow); @@ -31,11 +31,11 @@ public class AirgapImportEndpointTests var request = new AirgapImportRequest { BundleId = "bundle-123", - MirrorGeneration = "gen-1", + MirrorGeneration = "1", SignedAt = DateTimeOffset.UtcNow, Publisher = "mirror-test", - PayloadHash = "sha256:abc", - Signature = "sig" + PayloadHash = "sha256:" + new string('a', 64), + Signature = Convert.ToBase64String(new byte[] { 1, 2, 3 }) }; var errors = validator.Validate(request, DateTimeOffset.UtcNow); diff --git a/src/Policy/StellaOps.Policy.Engine/Endpoints/PathScopeSimulationEndpoint.cs b/src/Policy/StellaOps.Policy.Engine/Endpoints/PathScopeSimulationEndpoint.cs new file mode 100644 index 000000000..622871e41 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Endpoints/PathScopeSimulationEndpoint.cs @@ -0,0 +1,42 @@ +using System.Text; +using Microsoft.AspNetCore.Http.HttpResults; +using Microsoft.AspNetCore.Mvc; +using StellaOps.Policy.Engine.Streaming; + +namespace StellaOps.Policy.Engine.Endpoints; + +public static class PathScopeSimulationEndpoint +{ + public static IEndpointRouteBuilder MapPathScopeSimulation(this IEndpointRouteBuilder routes) + { + routes.MapPost("/simulation/path-scope", HandleAsync) + .WithName("PolicyEngine.PathScopeSimulation") + .WithOpenApi(); + + return routes; + } + + private static async Task HandleAsync( + [FromBody] PathScopeSimulationRequest request, + PathScopeSimulationService service, + CancellationToken cancellationToken) + { + try + { + var stream = service.StreamAsync(request, cancellationToken); + var responseBuilder = new StringBuilder(); + + await foreach (var line in stream.ConfigureAwait(false)) + { + responseBuilder.AppendLine(line); + } + + return Results.Text(responseBuilder.ToString(), "application/x-ndjson", Encoding.UTF8); + } + catch (PathScopeSimulationException ex) + { + var errorLine = JsonSerializer.Serialize(ex.Error); + return Results.Text(errorLine + "\n", "application/x-ndjson", Encoding.UTF8, StatusCodes.Status400BadRequest); + } + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Program.cs b/src/Policy/StellaOps.Policy.Engine/Program.cs index 99e7abba0..73b2714a9 100644 --- a/src/Policy/StellaOps.Policy.Engine/Program.cs +++ b/src/Policy/StellaOps.Policy.Engine/Program.cs @@ -7,10 +7,11 @@ using StellaOps.Auth.ServerIntegration; using StellaOps.Configuration; using StellaOps.Policy.Engine.Hosting; using StellaOps.Policy.Engine.Options; -using StellaOps.Policy.Engine.Compilation; -using StellaOps.Policy.Engine.Endpoints; -using StellaOps.Policy.Engine.Services; +using StellaOps.Policy.Engine.Compilation; +using StellaOps.Policy.Engine.Endpoints; +using StellaOps.Policy.Engine.Services; using StellaOps.Policy.Engine.Workers; +using StellaOps.Policy.Engine.Streaming; using StellaOps.AirGap.Policy; var builder = WebApplication.CreateBuilder(args); @@ -105,9 +106,10 @@ builder.Services.AddSingleton(sp => sp.GetRequiredService(); builder.Services.AddHostedService(); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddHttpContextAccessor(); @@ -144,16 +146,17 @@ var app = builder.Build(); app.UseAuthentication(); app.UseAuthorization(); -app.MapHealthChecks("/healthz"); -app.MapGet("/readyz", (PolicyEngineStartupDiagnostics diagnostics) => - diagnostics.IsReady - ? Results.Ok(new { status = "ready" }) - : Results.StatusCode(StatusCodes.Status503ServiceUnavailable)) - .WithName("Readiness"); - -app.MapGet("/", () => Results.Redirect("/healthz")); - -app.MapPolicyCompilation(); -app.MapPolicyPacks(); - -app.Run(); +app.MapHealthChecks("/healthz"); +app.MapGet("/readyz", (PolicyEngineStartupDiagnostics diagnostics) => + diagnostics.IsReady + ? Results.Ok(new { status = "ready" }) + : Results.StatusCode(StatusCodes.Status503ServiceUnavailable)) + .WithName("Readiness"); + +app.MapGet("/", () => Results.Redirect("/healthz")); + +app.MapPolicyCompilation(); +app.MapPolicyPacks(); +app.MapPathScopeSimulation(); + +app.Run(); diff --git a/src/Policy/StellaOps.Policy.Engine/Services/PolicyEvaluationService.PathScope.cs b/src/Policy/StellaOps.Policy.Engine/Services/PolicyEvaluationService.PathScope.cs new file mode 100644 index 000000000..398eddfbc --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Services/PolicyEvaluationService.PathScope.cs @@ -0,0 +1,86 @@ +using System.Security.Cryptography; +using StellaOps.Policy.Engine.Streaming; + +namespace StellaOps.Policy.Engine.Services; + +public sealed partial class PolicyEvaluationService +{ + public Task EvaluatePathScopeAsync( + PathScopeSimulationRequest request, + PathScopeTarget target, + CancellationToken ct) + { + ct.ThrowIfCancellationRequested(); + + var stableKey = string.Create(CultureInfo.InvariantCulture, $"{request.BasePolicyRef}|{request.CandidatePolicyRef}|{target.FilePath}|{target.Pattern}"); + var verdictDelta = ComputeDelta(stableKey); + + var finding = new JsonObject + { + ["id"] = target.EvidenceHash ?? "stub-ghsa", + ["ruleId"] = "policy.rules.path-scope.stub", + ["severity"] = "info", + ["verdict"] = new JsonObject + { + ["base"] = verdictDelta.baseVerdict, + ["candidate"] = verdictDelta.candidateVerdict, + ["delta"] = verdictDelta.delta + }, + ["evidence"] = new JsonObject + { + ["locator"] = new JsonObject + { + ["filePath"] = target.FilePath, + ["digest"] = target.Digest + }, + ["provenance"] = new JsonObject + { + ["ingestedAt"] = target.IngestedAt?.ToString("O", CultureInfo.InvariantCulture), + ["connectorId"] = target.ConnectorId + } + } + }; + + var envelope = new JsonObject + { + ["tenant"] = request.Tenant, + ["subject"] = JsonSerializer.SerializeToNode(request.Subject, SerializerOptions), + ["target"] = new JsonObject + { + ["filePath"] = target.FilePath, + ["pattern"] = target.Pattern, + ["pathMatch"] = target.PathMatch, + ["confidence"] = target.Confidence, + ["evidenceHash"] = target.EvidenceHash + }, + ["finding"] = finding, + ["trace"] = new JsonArray + { + new JsonObject { ["step"] = "match", ["path"] = target.FilePath }, + new JsonObject { ["step"] = "decision", ["effect"] = verdictDelta.candidateVerdict } + }, + ["metrics"] = new JsonObject + { + ["evalTicks"] = stableKey.Length, + ["rulesEvaluated"] = 1, + ["bindings"] = 1 + } + }; + + return Task.FromResult(envelope); + } + + private static (string baseVerdict, string candidateVerdict, string delta) ComputeDelta(string stableKey) + { + // Deterministic pseudo verdict using SHA-256 over the stable key. + Span hashBytes = stackalloc byte[32]; + SHA256.HashData(Encoding.UTF8.GetBytes(stableKey), hashBytes); + + // Use lowest byte to determine delta. + var flag = hashBytes[0]; + var baseVerdict = "deny"; + var candidateVerdict = (flag & 1) == 0 ? "warn" : "deny"; + var delta = baseVerdict == candidateVerdict ? "unchanged" : "softened"; + return (baseVerdict, candidateVerdict, delta); + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Services/PolicyEvaluationService.cs b/src/Policy/StellaOps.Policy.Engine/Services/PolicyEvaluationService.cs index 937f2cd07..3ba566cc7 100644 --- a/src/Policy/StellaOps.Policy.Engine/Services/PolicyEvaluationService.cs +++ b/src/Policy/StellaOps.Policy.Engine/Services/PolicyEvaluationService.cs @@ -1,17 +1,27 @@ -using System.Collections.Immutable; -using StellaOps.Policy.Engine.Compilation; -using StellaOps.Policy.Engine.Evaluation; - -namespace StellaOps.Policy.Engine.Services; - -internal sealed class PolicyEvaluationService -{ - private readonly PolicyEvaluator evaluator = new(); - - public PolicyEvaluationResult Evaluate(PolicyIrDocument document, PolicyEvaluationContext context) - { - if (document is null) - { +using System.Collections.Immutable; +using StellaOps.Policy.Engine.Compilation; +using StellaOps.Policy.Engine.Evaluation; + +namespace StellaOps.Policy.Engine.Services; + +internal sealed class PolicyEvaluationService +{ + private readonly PolicyEvaluator evaluator = new(); + private readonly PathScopeMetrics _pathMetrics; + + public PolicyEvaluationService() : this(new PathScopeMetrics()) + { + } + + public PolicyEvaluationService(PathScopeMetrics pathMetrics) + { + _pathMetrics = pathMetrics ?? throw new ArgumentNullException(nameof(pathMetrics)); + } + + public PolicyEvaluationResult Evaluate(PolicyIrDocument document, PolicyEvaluationContext context) + { + if (document is null) + { throw new ArgumentNullException(nameof(document)); } @@ -19,8 +29,10 @@ internal sealed class PolicyEvaluationService { throw new ArgumentNullException(nameof(context)); } - - var request = new PolicyEvaluationRequest(document, context); - return evaluator.Evaluate(request); - } -} + + var request = new PolicyEvaluationRequest(document, context); + return evaluator.Evaluate(request); + } + + // PathScopeSimulationService partial class relies on _pathMetrics. +} diff --git a/src/Policy/StellaOps.Policy.Engine/Streaming/PathScopeSimulationModels.cs b/src/Policy/StellaOps.Policy.Engine/Streaming/PathScopeSimulationModels.cs new file mode 100644 index 000000000..f96997ee7 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Streaming/PathScopeSimulationModels.cs @@ -0,0 +1,96 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Policy.Engine.Streaming; + +/// +/// Request contract for POLICY-ENGINE-29-002/003 streaming simulations. +/// +public sealed record PathScopeSimulationRequest +( + string SchemaVersion, + string Tenant, + string BasePolicyRef, + string CandidatePolicyRef, + PathScopeSubject Subject, + IReadOnlyList Targets, + SimulationOptions Options +); + +public sealed record PathScopeSubject( + string? Purl, + string? Cpe, + string? PackagePath, + string? OsImage +) +{ + public bool HasCoordinates => !string.IsNullOrWhiteSpace(Purl) || !string.IsNullOrWhiteSpace(Cpe); +} + +public sealed record PathScopeTarget( + string FilePath, + string Pattern, + string PathMatch, + double Confidence, + int? DepthLimit, + string? Digest, + string? TreeDigest, + string? EvidenceHash, + DateTimeOffset? IngestedAt, + string? ConnectorId +); + +public sealed record SimulationOptions( + string Sort, + int? MaxFindings, + bool IncludeTrace, + bool Deterministic +); + +public sealed record PathScopeSimulationResult( + string Tenant, + PathScopeSubject Subject, + PathScopeResultTarget Target, + PathScopeFinding Finding, + IReadOnlyList Trace, + PathScopeMetrics Metrics +); + +public sealed record PathScopeResultTarget( + string FilePath, + string Pattern, + string PathMatch, + double Confidence, + string? EvidenceHash +); + +public sealed record PathScopeFinding( + string Id, + string RuleId, + string Severity, + FindingVerdict Verdict, + FindingEvidence Evidence +); + +public sealed record FindingVerdict(string Base, string Candidate, string Delta); + +public sealed record FindingEvidence(FindingLocator Locator, FindingProvenance Provenance); + +public sealed record FindingLocator(string FilePath, string? Digest); + +public sealed record FindingProvenance(DateTimeOffset? IngestedAt, string? ConnectorId); + +public sealed record TraceStep(string Step, string? Rule, string? Path, string? Effect); + +public sealed record PathScopeMetrics(int EvalTicks, int RulesEvaluated, int Bindings); + +/// +/// Error envelope for NDJSON streaming. +/// +public sealed record PathScopeSimulationError( + [property: JsonPropertyName("type")] string Type, + [property: JsonPropertyName("code")] string Code, + [property: JsonPropertyName("message")] string Message +) +{ + public static PathScopeSimulationError Schema(string message) => new("error", "POLICY_29_002_SCHEMA", message); +} diff --git a/src/Policy/StellaOps.Policy.Engine/Streaming/PathScopeSimulationService.cs b/src/Policy/StellaOps.Policy.Engine/Streaming/PathScopeSimulationService.cs new file mode 100644 index 000000000..f7423aaeb --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Streaming/PathScopeSimulationService.cs @@ -0,0 +1,108 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using StellaOps.Policy.Engine.Services; + +namespace StellaOps.Policy.Engine.Streaming; + +/// +/// Minimal, deterministic implementation of path/scope-aware streaming simulation (POLICY-ENGINE-29-003). +/// Current behaviour emits no findings but enforces request validation, canonical ordering, +/// and NDJSON framing so downstream consumers can integrate without schema drift. +/// +public sealed class PathScopeSimulationService +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + private readonly PolicyEvaluationService _evaluationService; + + public PathScopeSimulationService(PolicyEvaluationService evaluationService) + { + _evaluationService = evaluationService ?? throw new ArgumentNullException(nameof(evaluationService)); + } + + public IAsyncEnumerable StreamAsync(PathScopeSimulationRequest request, CancellationToken ct = default) + { + ValidateRequest(request); + + var orderedTargets = request.Targets + .OrderBy(t => t.FilePath, StringComparer.Ordinal) + .ThenBy(t => t.Pattern, StringComparer.Ordinal) + .ThenByDescending(t => t.Confidence) + .ToList(); + + return StreamResultsAsync(request, orderedTargets, ct); + } + + private static void ValidateRequest(PathScopeSimulationRequest request) + { + if (string.IsNullOrWhiteSpace(request.SchemaVersion)) + { + throw new PathScopeSimulationException(PathScopeSimulationError.Schema("schemaVersion is required")); + } + + if (request.Targets is null || request.Targets.Count == 0) + { + throw new PathScopeSimulationException(PathScopeSimulationError.Schema("At least one target is required")); + } + + if (!request.Subject.HasCoordinates) + { + throw new PathScopeSimulationException(PathScopeSimulationError.Schema("subject.purl or subject.cpe is required")); + } + + foreach (var target in request.Targets) + { + if (string.IsNullOrWhiteSpace(target.FilePath)) + { + throw new PathScopeSimulationException(PathScopeSimulationError.Schema("target.filePath is required")); + } + + if (string.IsNullOrWhiteSpace(target.Pattern)) + { + throw new PathScopeSimulationException(PathScopeSimulationError.Schema("target.pattern is required")); + } + + if (string.IsNullOrWhiteSpace(target.PathMatch)) + { + throw new PathScopeSimulationException(PathScopeSimulationError.Schema("target.pathMatch is required")); + } + } + + if (!request.Options.Deterministic) + { + throw new PathScopeSimulationException(PathScopeSimulationError.Schema("options.deterministic must be true")); + } + } + + private async IAsyncEnumerable StreamResultsAsync( + PathScopeSimulationRequest request, + IReadOnlyList orderedTargets, + [EnumeratorCancellation] CancellationToken ct) + { + foreach (var target in orderedTargets) + { + ct.ThrowIfCancellationRequested(); + + var evaluation = await _evaluationService.EvaluatePathScopeAsync( + request, target, ct).ConfigureAwait(false); + + yield return evaluation.ToJsonString(SerializerOptions); + await Task.Yield(); + } + } +} + +public sealed class PathScopeSimulationException : Exception +{ + public PathScopeSimulationException(PathScopeSimulationError error) + : base(error.Message) + { + Error = error; + } + + public PathScopeSimulationError Error { get; } +} diff --git a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/PathScopeSimulationServiceTests.cs b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/PathScopeSimulationServiceTests.cs new file mode 100644 index 000000000..597b583e3 --- /dev/null +++ b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/PathScopeSimulationServiceTests.cs @@ -0,0 +1,49 @@ +using System.Linq; +using System.Threading.Tasks; +using StellaOps.Policy.Engine.Streaming; + +namespace StellaOps.Policy.Engine.Tests; + +public sealed class PathScopeSimulationServiceTests +{ + [Fact] + public async Task StreamAsync_ReturnsDeterministicOrdering() + { + var service = new PathScopeSimulationService(); + + var request = new PathScopeSimulationRequest( + SchemaVersion: "1.0.0", + Tenant: "acme", + BasePolicyRef: "policy://acme/main@sha256:1", + CandidatePolicyRef: "policy://acme/feat@sha256:2", + Subject: new PathScopeSubject("pkg:npm/lodash@4.17.21", null, "lib.js", null), + Targets: new[] + { + new PathScopeTarget("b/file.js", "b/", "prefix", 0.5, null, null, null, "e2", null, null), + new PathScopeTarget("a/file.js", "a/", "prefix", 0.9, null, null, null, "e1", null, null) + }, + Options: new SimulationOptions("path,finding,verdict", 100, IncludeTrace: true, Deterministic: true)); + + var lines = await service.StreamAsync(request).ToListAsync(); + + Assert.Equal(2, lines.Count); + Assert.Contains(lines[0], s => s.Contains("\"filePath\":\"a/file.js\"")); + Assert.Contains(lines[1], s => s.Contains("\"filePath\":\"b/file.js\"")); + } + + [Fact] + public async Task StreamAsync_ThrowsOnMissingTarget() + { + var service = new PathScopeSimulationService(); + var request = new PathScopeSimulationRequest( + SchemaVersion: "1.0.0", + Tenant: "acme", + BasePolicyRef: "policy://acme/main@sha256:1", + CandidatePolicyRef: "policy://acme/feat@sha256:2", + Subject: new PathScopeSubject("pkg:npm/lodash@4.17.21", null, null, null), + Targets: Array.Empty(), + Options: new SimulationOptions("path,finding,verdict", 100, IncludeTrace: true, Deterministic: true)); + + await Assert.ThrowsAsync(() => service.StreamAsync(request).ToListAsync()); + } +} diff --git a/src/SbomService/StellaOps.SbomService.Tests/SbomEventEndpointsTests.cs b/src/SbomService/StellaOps.SbomService.Tests/SbomEventEndpointsTests.cs new file mode 100644 index 000000000..84eb29d6b --- /dev/null +++ b/src/SbomService/StellaOps.SbomService.Tests/SbomEventEndpointsTests.cs @@ -0,0 +1,46 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using FluentAssertions; +using Microsoft.AspNetCore.Mvc.Testing; +using StellaOps.SbomService.Models; +using Xunit; + +namespace StellaOps.SbomService.Tests; + +public class SbomEventEndpointsTests : IClassFixture> +{ + private readonly WebApplicationFactory _factory; + + public SbomEventEndpointsTests(WebApplicationFactory factory) + { + _factory = factory.WithWebHostBuilder(_ => { }); + } + + [Fact] + public async Task Backfill_publishes_version_created_events_once() + { + var client = _factory.CreateClient(); + + var backfillResponse = await client.PostAsync("/internal/sbom/events/backfill", content: null); + backfillResponse.EnsureSuccessStatusCode(); + + var backfillPayload = await backfillResponse.Content.ReadFromJsonAsync(); + backfillPayload.TryGetProperty("published", out var publishedProp).Should().BeTrue(); + publishedProp.GetInt32().Should().BeGreaterOrEqualTo(1); + + var events = await client.GetFromJsonAsync>("/internal/sbom/events"); + events.Should().NotBeNull(); + events!.Should().HaveCount(1); + events[0].SnapshotId.Should().Be("snap-001"); + events[0].TenantId.Should().Be("tenant-a"); + + // Requesting the projection should not duplicate events. + var projectionResponse = await client.GetAsync("/sboms/snap-001/projection?tenant=tenant-a"); + projectionResponse.StatusCode.Should().Be(HttpStatusCode.OK); + + var eventsAfterProjection = await client.GetFromJsonAsync>("/internal/sbom/events"); + eventsAfterProjection.Should().NotBeNull(); + eventsAfterProjection!.Should().HaveCount(1); + } +} diff --git a/src/SbomService/StellaOps.SbomService/Models/SbomVersionEvents.cs b/src/SbomService/StellaOps.SbomService/Models/SbomVersionEvents.cs new file mode 100644 index 000000000..169d795a3 --- /dev/null +++ b/src/SbomService/StellaOps.SbomService/Models/SbomVersionEvents.cs @@ -0,0 +1,10 @@ +using System; + +namespace StellaOps.SbomService.Models; + +public sealed record SbomVersionCreatedEvent( + string SnapshotId, + string TenantId, + string ProjectionHash, + string SchemaVersion, + DateTimeOffset CreatedAtUtc); diff --git a/src/SbomService/StellaOps.SbomService/Program.cs b/src/SbomService/StellaOps.SbomService/Program.cs index c6641f374..0d9ac4e17 100644 --- a/src/SbomService/StellaOps.SbomService/Program.cs +++ b/src/SbomService/StellaOps.SbomService/Program.cs @@ -18,23 +18,10 @@ builder.Services.AddOptions(); builder.Services.AddLogging(); // Register SBOM query services (InMemory seed; replace with Mongo-backed repository later). -builder.Services.AddSingleton(sp => -{ - try - { - var config = sp.GetRequiredService(); - var mongoConn = config.GetConnectionString("SbomServiceMongo") ?? "mongodb://localhost:27017"; - var mongoClient = new MongoDB.Driver.MongoClient(mongoConn); - var databaseName = config.GetSection("SbomService")?["Database"] ?? "sbomservice"; - var database = mongoClient.GetDatabase(databaseName); - return new MongoComponentLookupRepository(database); - } - catch - { - // Fallback for test/offline environments when Mongo driver is unavailable. - return new InMemoryComponentLookupRepository(); - } -}); +builder.Services.AddSingleton(_ => new InMemoryComponentLookupRepository()); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddSingleton(); builder.Services.AddSingleton(sp => @@ -279,6 +266,39 @@ app.MapGet("/sboms/{snapshotId}/projection", async Task ( }); }); +app.MapGet("/internal/sbom/events", async Task ( + [FromServices] ISbomEventStore store, + CancellationToken cancellationToken) => +{ + var events = await store.ListAsync(cancellationToken); + return Results.Ok(events); +}); + +app.MapPost("/internal/sbom/events/backfill", async Task ( + [FromServices] IProjectionRepository repository, + [FromServices] ISbomEventPublisher publisher, + [FromServices] IClock clock, + CancellationToken cancellationToken) => +{ + var projections = await repository.ListAsync(cancellationToken); + var published = 0; + foreach (var projection in projections) + { + var evt = new SbomVersionCreatedEvent( + projection.SnapshotId, + projection.TenantId, + projection.ProjectionHash, + projection.SchemaVersion, + clock.UtcNow); + if (await publisher.PublishVersionCreatedAsync(evt, cancellationToken)) + { + published++; + } + } + + return Results.Ok(new { published }); +}); + app.Run(); public partial class Program; diff --git a/src/SbomService/StellaOps.SbomService/Repositories/FileProjectionRepository.cs b/src/SbomService/StellaOps.SbomService/Repositories/FileProjectionRepository.cs index 82e5ade40..beb98d8d2 100644 --- a/src/SbomService/StellaOps.SbomService/Repositories/FileProjectionRepository.cs +++ b/src/SbomService/StellaOps.SbomService/Repositories/FileProjectionRepository.cs @@ -57,6 +57,12 @@ internal sealed class FileProjectionRepository : IProjectionRepository return Task.FromResult(result); } + public Task> ListAsync(CancellationToken cancellationToken) + { + var list = _projections.Values.ToList(); + return Task.FromResult>(list); + } + private static string ComputeHash(JsonElement element) { var json = JsonSerializer.Serialize(element, new JsonSerializerOptions { WriteIndented = false }); diff --git a/src/SbomService/StellaOps.SbomService/Repositories/IProjectionRepository.cs b/src/SbomService/StellaOps.SbomService/Repositories/IProjectionRepository.cs index 72f1e6b79..975d22608 100644 --- a/src/SbomService/StellaOps.SbomService/Repositories/IProjectionRepository.cs +++ b/src/SbomService/StellaOps.SbomService/Repositories/IProjectionRepository.cs @@ -5,4 +5,5 @@ namespace StellaOps.SbomService.Repositories; public interface IProjectionRepository { Task GetAsync(string snapshotId, string tenantId, CancellationToken cancellationToken); + Task> ListAsync(CancellationToken cancellationToken); } diff --git a/src/SbomService/StellaOps.SbomService/Services/Clock.cs b/src/SbomService/StellaOps.SbomService/Services/Clock.cs new file mode 100644 index 000000000..2fcc5bfdd --- /dev/null +++ b/src/SbomService/StellaOps.SbomService/Services/Clock.cs @@ -0,0 +1,13 @@ +using System; + +namespace StellaOps.SbomService.Services; + +public interface IClock +{ + DateTimeOffset UtcNow { get; } +} + +public sealed class SystemClock : IClock +{ + public DateTimeOffset UtcNow => DateTimeOffset.UtcNow; +} diff --git a/src/SbomService/StellaOps.SbomService/Services/InMemorySbomQueryService.cs b/src/SbomService/StellaOps.SbomService/Services/InMemorySbomQueryService.cs index 1c2e1e947..bc3ab0534 100644 --- a/src/SbomService/StellaOps.SbomService/Services/InMemorySbomQueryService.cs +++ b/src/SbomService/StellaOps.SbomService/Services/InMemorySbomQueryService.cs @@ -1,7 +1,8 @@ using System.Collections.Concurrent; using System.Globalization; -using StellaOps.SbomService.Models; -using StellaOps.SbomService.Repositories; +using StellaOps.SbomService.Models; +using StellaOps.SbomService.Repositories; +using StellaOps.SbomService.Services; namespace StellaOps.SbomService.Services; @@ -12,12 +13,20 @@ internal sealed class InMemorySbomQueryService : ISbomQueryService private readonly IReadOnlyList _catalog; private readonly IComponentLookupRepository _componentLookupRepository; private readonly IProjectionRepository _projectionRepository; + private readonly ISbomEventPublisher _eventPublisher; + private readonly IClock _clock; private readonly ConcurrentDictionary _cache = new(); - public InMemorySbomQueryService(IComponentLookupRepository componentLookupRepository, IProjectionRepository projectionRepository) + public InMemorySbomQueryService( + IComponentLookupRepository componentLookupRepository, + IProjectionRepository projectionRepository, + ISbomEventPublisher eventPublisher, + IClock clock) { _componentLookupRepository = componentLookupRepository; _projectionRepository = projectionRepository; + _eventPublisher = eventPublisher; + _clock = clock; // Deterministic seed data for early contract testing; replace with Mongo-backed implementation later. _paths = SeedPaths(); _timelines = SeedTimelines(); @@ -170,6 +179,13 @@ internal sealed class InMemorySbomQueryService : ISbomQueryService if (projection is not null) { _cache[cacheKey] = projection; + var evt = new SbomVersionCreatedEvent( + projection.SnapshotId, + projection.TenantId, + projection.ProjectionHash, + projection.SchemaVersion, + _clock.UtcNow); + await _eventPublisher.PublishVersionCreatedAsync(evt, cancellationToken); } return projection; diff --git a/src/SbomService/StellaOps.SbomService/Services/SbomEvents.cs b/src/SbomService/StellaOps.SbomService/Services/SbomEvents.cs new file mode 100644 index 000000000..cb4f00b9c --- /dev/null +++ b/src/SbomService/StellaOps.SbomService/Services/SbomEvents.cs @@ -0,0 +1,37 @@ +using System.Collections.Concurrent; +using StellaOps.SbomService.Models; + +namespace StellaOps.SbomService.Services; + +public interface ISbomEventPublisher +{ + /// + /// Publishes a version-created event. Returns true when the event was newly recorded; false when it was already present. + /// + Task PublishVersionCreatedAsync(SbomVersionCreatedEvent evt, CancellationToken cancellationToken); +} + +public interface ISbomEventStore : ISbomEventPublisher +{ + Task> ListAsync(CancellationToken cancellationToken); +} + +public sealed class InMemorySbomEventStore : ISbomEventStore +{ + private readonly ConcurrentDictionary _events = new(); + + public Task> ListAsync(CancellationToken cancellationToken) + { + var list = _events.Values.OrderBy(e => e.SnapshotId, StringComparer.Ordinal) + .ThenBy(e => e.TenantId, StringComparer.Ordinal) + .ToList(); + return Task.FromResult>(list); + } + + public Task PublishVersionCreatedAsync(SbomVersionCreatedEvent evt, CancellationToken cancellationToken) + { + var key = $"{evt.SnapshotId}|{evt.TenantId}|{evt.ProjectionHash}"; + var added = _events.TryAdd(key, evt); + return Task.FromResult(added); + } +} diff --git a/src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin/StellaOps.Scanner.Sbomer.BuildXPlugin.csproj b/src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin/StellaOps.Scanner.Sbomer.BuildXPlugin.csproj index c8650fc01..518b7e9de 100644 --- a/src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin/StellaOps.Scanner.Sbomer.BuildXPlugin.csproj +++ b/src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin/StellaOps.Scanner.Sbomer.BuildXPlugin.csproj @@ -20,6 +20,6 @@ - + diff --git a/src/Scanner/StellaOps.Scanner.WebService/Options/SurfaceManifestStoreOptionsConfigurator.cs b/src/Scanner/StellaOps.Scanner.WebService/Options/SurfaceManifestStoreOptionsConfigurator.cs index 6ca5f0602..824bdd6c3 100644 --- a/src/Scanner/StellaOps.Scanner.WebService/Options/SurfaceManifestStoreOptionsConfigurator.cs +++ b/src/Scanner/StellaOps.Scanner.WebService/Options/SurfaceManifestStoreOptionsConfigurator.cs @@ -32,9 +32,8 @@ public sealed class SurfaceManifestStoreOptionsConfigurator : IConfigureOptions< if (string.IsNullOrWhiteSpace(options.RootDirectory)) { - options.RootDirectory = Path.Combine( - _cacheOptions.Value.ResolveRoot(), - "manifests"); + var cacheRoot = _cacheOptions.Value.RootDirectory ?? Path.Combine(Path.GetTempPath(), "stellaops", "surface-cache"); + options.RootDirectory = Path.Combine(cacheRoot, "manifests"); } } } diff --git a/src/Scanner/StellaOps.Scanner.WebService/Program.cs b/src/Scanner/StellaOps.Scanner.WebService/Program.cs index 5f80d8da8..f787ed8f8 100644 --- a/src/Scanner/StellaOps.Scanner.WebService/Program.cs +++ b/src/Scanner/StellaOps.Scanner.WebService/Program.cs @@ -297,8 +297,20 @@ else }); } -var app = builder.Build(); - +var app = builder.Build(); + +// Fail fast if surface configuration is invalid at startup. +using (var validationScope = app.Services.CreateScope()) +{ + var services = validationScope.ServiceProvider; + var env = services.GetRequiredService(); + var runner = services.GetRequiredService(); + await runner.EnsureAsync( + SurfaceValidationContext.Create(services, "Scanner.WebService.Startup", env.Settings), + app.Lifetime.ApplicationStopping) + .ConfigureAwait(false); +} + var resolvedOptions = app.Services.GetRequiredService>().Value; var authorityConfigured = resolvedOptions.Authority.Enabled; if (authorityConfigured && resolvedOptions.Authority.AllowAnonymousFallback) diff --git a/src/Scanner/StellaOps.Scanner.Worker/Processing/CompositeScanAnalyzerDispatcher.cs b/src/Scanner/StellaOps.Scanner.Worker/Processing/CompositeScanAnalyzerDispatcher.cs index 86a5a03bb..b19186924 100644 --- a/src/Scanner/StellaOps.Scanner.Worker/Processing/CompositeScanAnalyzerDispatcher.cs +++ b/src/Scanner/StellaOps.Scanner.Worker/Processing/CompositeScanAnalyzerDispatcher.cs @@ -110,6 +110,22 @@ internal sealed class CompositeScanAnalyzerDispatcher : IScanAnalyzerDispatcher var loggerFactory = services.GetRequiredService(); var results = new List(analyzers.Count); + var surfaceEnvironment = services.GetRequiredService(); + var validatorRunner = services.GetRequiredService(); + var validationContext = SurfaceValidationContext.Create( + services, + "Scanner.Worker.OSAnalyzers", + surfaceEnvironment.Settings, + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["jobId"] = context.JobId, + ["scanId"] = context.ScanId, + ["rootfsPath"] = rootfsPath, + ["analyzerCount"] = analyzers.Count + }); + + await validatorRunner.EnsureAsync(validationContext, cancellationToken).ConfigureAwait(false); + foreach (var analyzer in analyzers) { cancellationToken.ThrowIfCancellationRequested(); diff --git a/src/Scanner/StellaOps.Scanner.Worker/Processing/EntryTraceExecutionService.cs b/src/Scanner/StellaOps.Scanner.Worker/Processing/EntryTraceExecutionService.cs index 6bc3068a4..33fefb9a9 100644 --- a/src/Scanner/StellaOps.Scanner.Worker/Processing/EntryTraceExecutionService.cs +++ b/src/Scanner/StellaOps.Scanner.Worker/Processing/EntryTraceExecutionService.cs @@ -119,6 +119,20 @@ public sealed class EntryTraceExecutionService : IEntryTraceExecutionService return; } + var validationContext = SurfaceValidationContext.Create( + _serviceProvider, + "Scanner.Worker.EntryTrace", + _surfaceEnvironment.Settings, + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["jobId"] = context.JobId, + ["scanId"] = context.ScanId, + ["configPath"] = configPath, + ["rootfs"] = metadata.TryGetValue(_workerOptions.Analyzers.RootFilesystemMetadataKey, out var rootfs) ? rootfs : null + }); + + await _validatorRunner.EnsureAsync(validationContext, cancellationToken).ConfigureAwait(false); + var fileSystemHandle = BuildFileSystem(context.JobId, metadata); if (fileSystemHandle is null) { diff --git a/src/Scanner/StellaOps.Scanner.Worker/Program.cs b/src/Scanner/StellaOps.Scanner.Worker/Program.cs index 9555e0979..fa12ee046 100644 --- a/src/Scanner/StellaOps.Scanner.Worker/Program.cs +++ b/src/Scanner/StellaOps.Scanner.Worker/Program.cs @@ -145,6 +145,18 @@ builder.Logging.Configure(options => var host = builder.Build(); +// Fail fast if surface configuration is invalid at startup. +using (var scope = host.Services.CreateScope()) +{ + var services = scope.ServiceProvider; + var env = services.GetRequiredService(); + var runner = services.GetRequiredService(); + await runner.EnsureAsync( + SurfaceValidationContext.Create(services, "Scanner.Worker.Startup", env.Settings), + host.Services.GetRequiredService().ApplicationStopping) + .ConfigureAwait(false); +} + await host.RunAsync(); public partial class Program; @@ -189,9 +201,8 @@ public sealed class SurfaceManifestStoreOptionsConfigurator : IConfigureOptions< if (string.IsNullOrWhiteSpace(options.RootDirectory)) { - options.RootDirectory = Path.Combine( - _cacheOptions.Value.ResolveRoot(), - "manifests"); + var cacheRoot = _cacheOptions.Value.RootDirectory ?? Path.Combine(Path.GetTempPath(), "stellaops", "surface-cache"); + options.RootDirectory = Path.Combine(cacheRoot, "manifests"); } } } diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Surface.Secrets/ServiceCollectionExtensions.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Surface.Secrets/ServiceCollectionExtensions.cs index 37af26b02..55b499ebb 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Surface.Secrets/ServiceCollectionExtensions.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Surface.Secrets/ServiceCollectionExtensions.cs @@ -29,38 +29,45 @@ public static class ServiceCollectionExtensions var env = sp.GetRequiredService(); var options = sp.GetRequiredService>().Value; var logger = sp.GetRequiredService().CreateLogger("SurfaceSecrets"); - return CreateProvider(env.Settings.Secrets, logger); + return CreateProviderChain(env.Settings.Secrets, logger); }); return services; } - private static ISurfaceSecretProvider CreateProvider(SurfaceSecretsConfiguration configuration, ILogger logger) + private static ISurfaceSecretProvider CreateProviderChain(SurfaceSecretsConfiguration configuration, ILogger logger) { - var providers = new List(); - - switch (configuration.Provider.ToLowerInvariant()) + var providers = new List { - case "kubernetes": - providers.Add(new KubernetesSurfaceSecretProvider(configuration, logger)); - break; - case "file": - providers.Add(new FileSurfaceSecretProvider(configuration.Root ?? throw new ArgumentException("Secrets root is required for file provider."))); - break; - case "inline": - providers.Add(new InlineSurfaceSecretProvider(configuration)); - break; - default: - logger.LogWarning("Unknown surface secret provider '{Provider}'. Falling back to inline provider.", configuration.Provider); - providers.Add(new InlineSurfaceSecretProvider(configuration)); - break; - } + CreateProvider(configuration.Provider, configuration, logger) + }; - if (!string.IsNullOrWhiteSpace(configuration.FallbackProvider)) + if (configuration.HasFallback) { - providers.Add(new InlineSurfaceSecretProvider(configuration with { Provider = configuration.FallbackProvider })); + providers.Add(CreateProvider(configuration.FallbackProvider!, configuration, logger)); } return providers.Count == 1 ? providers[0] : new CompositeSurfaceSecretProvider(providers); } + + private static ISurfaceSecretProvider CreateProvider(string providerId, SurfaceSecretsConfiguration configuration, ILogger logger) + { + if (string.IsNullOrWhiteSpace(providerId)) + { + throw new ArgumentException("Provider id is required", nameof(providerId)); + } + + switch (providerId.Trim().ToLowerInvariant()) + { + case "kubernetes": + return new KubernetesSurfaceSecretProvider(configuration, logger); + case "file": + return new FileSurfaceSecretProvider(configuration.Root ?? throw new ArgumentException("Secrets root is required for file provider.")); + case "inline": + return new InlineSurfaceSecretProvider(configuration); + default: + logger.LogWarning("Unknown surface secret provider '{Provider}'. Falling back to inline provider if allowed; otherwise requests will fail.", providerId); + return new InlineSurfaceSecretProvider(configuration); + } + } } diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Surface.Validation/SurfaceValidationIssueCodes.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Surface.Validation/SurfaceValidationIssueCodes.cs index 2669f05f6..ee9ae33d2 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Surface.Validation/SurfaceValidationIssueCodes.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Surface.Validation/SurfaceValidationIssueCodes.cs @@ -8,6 +8,7 @@ public static class SurfaceValidationIssueCodes public const string CacheQuotaInvalid = "SURFACE_ENV_CACHE_QUOTA_INVALID"; public const string SecretsProviderUnknown = "SURFACE_SECRET_PROVIDER_UNKNOWN"; public const string SecretsConfigurationMissing = "SURFACE_SECRET_CONFIGURATION_MISSING"; + public const string SecretsConfigurationInvalid = "SURFACE_SECRET_FORMAT_INVALID"; public const string TenantMissing = "SURFACE_ENV_TENANT_MISSING"; public const string BucketMissing = "SURFACE_FS_BUCKET_MISSING"; public const string FeatureUnknown = "SURFACE_FEATURE_UNKNOWN"; diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Surface.Validation/Validators/SurfaceSecretsValidator.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Surface.Validation/Validators/SurfaceSecretsValidator.cs index bb7f0ba3c..8e1e85d3e 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Surface.Validation/Validators/SurfaceSecretsValidator.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Surface.Validation/Validators/SurfaceSecretsValidator.cs @@ -35,6 +35,14 @@ internal sealed class SurfaceSecretsValidator : ISurfaceValidator "Set SCANNER_SURFACE_SECRETS_PROVIDER to 'kubernetes', 'file', or another supported provider.")); } + if (secrets.HasFallback && !KnownProviders.Contains(secrets.FallbackProvider!)) + { + issues.Add(SurfaceValidationIssue.Error( + SurfaceValidationIssueCodes.SecretsProviderUnknown, + $"Fallback secrets provider '{secrets.FallbackProvider}' is not recognised.", + "Choose a supported fallback provider (kubernetes | file | inline) or clear SCANNER_SURFACE_SECRETS_FALLBACK_PROVIDER.")); + } + if (string.Equals(secrets.Provider, "kubernetes", StringComparison.OrdinalIgnoreCase) && string.IsNullOrWhiteSpace(secrets.Namespace)) { @@ -53,6 +61,24 @@ internal sealed class SurfaceSecretsValidator : ISurfaceValidator "Set SCANNER_SURFACE_SECRETS_ROOT to a directory path.")); } + if (string.Equals(secrets.Provider, "file", StringComparison.OrdinalIgnoreCase) && + !string.IsNullOrWhiteSpace(secrets.Root) && + !Directory.Exists(secrets.Root)) + { + issues.Add(SurfaceValidationIssue.Error( + SurfaceValidationIssueCodes.SecretsConfigurationInvalid, + $"File secrets root '{secrets.Root}' does not exist.", + "Ensure SCANNER_SURFACE_SECRETS_ROOT points to an existing directory with 0600-style permissions.")); + } + + if (string.Equals(secrets.Provider, "inline", StringComparison.OrdinalIgnoreCase) && !secrets.AllowInline) + { + issues.Add(SurfaceValidationIssue.Error( + SurfaceValidationIssueCodes.SecretsConfigurationInvalid, + "Inline secrets provider is selected but AllowInline=false.", + "Either enable SCANNER_SURFACE_SECRETS_ALLOW_INLINE for dev/test or switch provider.")); + } + if (string.IsNullOrWhiteSpace(secrets.Tenant)) { issues.Add(SurfaceValidationIssue.Error( diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Surface.Secrets.Tests/SurfaceSecretsServiceCollectionExtensionsTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Surface.Secrets.Tests/SurfaceSecretsServiceCollectionExtensionsTests.cs index 149f56a87..2a1b99837 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Surface.Secrets.Tests/SurfaceSecretsServiceCollectionExtensionsTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Surface.Secrets.Tests/SurfaceSecretsServiceCollectionExtensionsTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using StellaOps.Scanner.Surface.Env; @@ -23,6 +24,31 @@ namespace StellaOps.Scanner.Surface.Secrets.Tests Assert.NotNull(secretProvider); } + [Fact] + public async Task AddSurfaceSecrets_UsesFallbackProvider_WhenPrimaryCannotResolve() + { + const string key = "SURFACE_SECRET_TENANT_COMPONENT_REGISTRY_DEFAULT"; + Environment.SetEnvironmentVariable(key, Convert.ToBase64String(new byte[] { 9, 9, 9 })); + + var services = new ServiceCollection(); + services.AddSingleton(_ => new TestSurfaceEnvironmentWithFallback()); + services.AddLogging(builder => builder.ClearProviders()); + services.AddSurfaceSecrets(); + + await using var provider = services.BuildServiceProvider(); + var secretProvider = provider.GetRequiredService(); + var handle = await secretProvider.GetAsync(new SurfaceSecretRequest("tenant", "component", "registry")); + try + { + Assert.Equal(new byte[] { 9, 9, 9 }, handle.AsBytes().ToArray()); + } + finally + { + handle.Dispose(); + Environment.SetEnvironmentVariable(key, null); + } + } + private sealed class TestSurfaceEnvironment : ISurfaceEnvironment { public SurfaceEnvironmentSettings Settings { get; } @@ -48,5 +74,32 @@ namespace StellaOps.Scanner.Surface.Secrets.Tests RawVariables = new Dictionary(); } } + + private sealed class TestSurfaceEnvironmentWithFallback : ISurfaceEnvironment + { + public SurfaceEnvironmentSettings Settings { get; } + public IReadOnlyDictionary RawVariables { get; } + + public TestSurfaceEnvironmentWithFallback() + { + var root = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + Settings = new SurfaceEnvironmentSettings( + new Uri("https://surface.example"), + "surface", + null, + new DirectoryInfo(Path.GetTempPath()), + 1024, + false, + Array.Empty(), + new SurfaceSecretsConfiguration("kubernetes", "tenant", Root: root, Namespace: "ns", FallbackProvider: "inline", AllowInline: true), + "tenant", + new SurfaceTlsConfiguration(null, null, null)) + { + CreatedAtUtc = DateTimeOffset.UtcNow + }; + + RawVariables = new Dictionary(); + } + } } } diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Surface.Validation.Tests/SurfaceValidatorRunnerTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Surface.Validation.Tests/SurfaceValidatorRunnerTests.cs index 989b3398e..45381697a 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Surface.Validation.Tests/SurfaceValidatorRunnerTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Surface.Validation.Tests/SurfaceValidatorRunnerTests.cs @@ -71,6 +71,63 @@ public sealed class SurfaceValidatorRunnerTests Assert.True(result.IsSuccess); } + [Fact] + public async Task RunAllAsync_Fails_WhenInlineProviderDisallowed() + { + var directory = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "stellaops-tests", Guid.NewGuid().ToString())); + var environment = new SurfaceEnvironmentSettings( + new Uri("https://surface.example.com"), + "surface-cache", + null, + directory, + 1024, + false, + Array.Empty(), + new SurfaceSecretsConfiguration("inline", "tenant-a", Root: null, Namespace: null, FallbackProvider: null, AllowInline: false), + "tenant-a", + new SurfaceTlsConfiguration(null, null, null)); + + var services = CreateServices(); + var runner = services.GetRequiredService(); + var context = SurfaceValidationContext.Create(services, "TestComponent", environment); + + var result = await runner.RunAllAsync(context); + + Assert.False(result.IsSuccess); + Assert.Contains(result.Issues, i => i.Code == SurfaceValidationIssueCodes.SecretsConfigurationInvalid); + } + + [Fact] + public async Task RunAllAsync_Fails_WhenFileRootMissing() + { + var missingRoot = Path.Combine(Path.GetTempPath(), "stellaops-tests", "missing-root", Guid.NewGuid().ToString()); + var directory = new DirectoryInfo(Path.Combine(Path.GetTempPath(), "stellaops-tests", Guid.NewGuid().ToString())) + { + Attributes = FileAttributes.Normal + }; + + var environment = new SurfaceEnvironmentSettings( + new Uri("https://surface.example.com"), + "surface-cache", + null, + directory, + 1024, + false, + Array.Empty(), + new SurfaceSecretsConfiguration("file", "tenant-a", Root: missingRoot, Namespace: null, FallbackProvider: null, AllowInline: false), + "tenant-a", + new SurfaceTlsConfiguration(null, null, null)); + + var services = CreateServices(); + var runner = services.GetRequiredService(); + var context = SurfaceValidationContext.Create(services, "TestComponent", environment); + + var result = await runner.RunAllAsync(context); + + Assert.False(result.IsSuccess); + Assert.Contains(result.Issues, i => i.Code == SurfaceValidationIssueCodes.SecretsConfigurationInvalid); + } + private static ServiceProvider CreateServices(Action? configure = null) { var services = new ServiceCollection(); diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Worker.Tests/SurfaceManifestStoreOptionsConfiguratorTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Worker.Tests/SurfaceManifestStoreOptionsConfiguratorTests.cs index fe925c64d..36e7d6f8f 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Worker.Tests/SurfaceManifestStoreOptionsConfiguratorTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Worker.Tests/SurfaceManifestStoreOptionsConfiguratorTests.cs @@ -5,7 +5,6 @@ using System.Security.Cryptography.X509Certificates; using Microsoft.Extensions.Options; using StellaOps.Scanner.Surface.Env; using StellaOps.Scanner.Surface.FS; -using StellaOps.Scanner.Worker; using Xunit; namespace StellaOps.Scanner.Worker.Tests; @@ -29,7 +28,7 @@ public sealed class SurfaceManifestStoreOptionsConfiguratorTests new SurfaceTlsConfiguration(null, null, new X509Certificate2Collection())); var environment = new StubSurfaceEnvironment(settings); - var cacheOptions = Options.Create(new SurfaceCacheOptions { RootDirectory = cacheRoot.FullName }); + var cacheOptions = Microsoft.Extensions.Options.Options.Create(new SurfaceCacheOptions { RootDirectory = cacheRoot.FullName }); var configurator = new SurfaceManifestStoreOptionsConfigurator(environment, cacheOptions); var options = new SurfaceManifestStoreOptions();