diff --git a/.gitea/workflows/console-ci.yml b/.gitea/workflows/console-ci.yml new file mode 100644 index 000000000..5a8fa961c --- /dev/null +++ b/.gitea/workflows/console-ci.yml @@ -0,0 +1,83 @@ +name: Console CI + +on: + push: + branches: [ main ] + paths: + - 'src/UI/**' + - '.gitea/workflows/console-ci.yml' + - 'docs/modules/devops/console-ci-contract.md' + pull_request: + branches: [ main, develop ] + paths: + - 'src/UI/**' + - '.gitea/workflows/console-ci.yml' + - 'docs/modules/devops/console-ci-contract.md' + +jobs: + console-ci: + runs-on: ubuntu-22.04 + env: + PNPM_HOME: ~/.pnpm + PLAYWRIGHT_BROWSERS_PATH: ./.playwright + SOURCE_DATE_EPOCH: ${{ github.run_id }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Node.js 20 + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Enable pnpm + run: | + corepack enable + corepack prepare pnpm@9 --activate + + - name: Cache pnpm store & node_modules + uses: actions/cache@v4 + with: + path: | + ~/.pnpm-store + node_modules + ./.pnpm-store + ./.playwright + key: console-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }} + + - name: Install dependencies (offline-first) + env: + PNPM_FETCH_RETRIES: 0 + PNPM_OFFLINE: 1 + run: | + pnpm install --frozen-lockfile || PNPM_OFFLINE=0 pnpm install --frozen-lockfile --prefer-offline + + - name: Lint / Types + run: pnpm lint && pnpm format:check && pnpm typecheck + + - name: Unit tests + run: pnpm test -- --runInBand --reporter=junit --outputFile=.artifacts/junit.xml + + - name: Storybook a11y + run: | + pnpm storybook:build + pnpm storybook:a11y --ci --output .artifacts/storybook-a11y.json + + - name: Playwright smoke + run: pnpm playwright test --config=playwright.config.ci.ts --reporter=list,junit=.artifacts/playwright.xml + + - name: Lighthouse (CI budgets) + run: | + pnpm serve --port 4173 & + pnpm lhci autorun --config=lighthouserc.ci.js --upload.target=filesystem --upload.outputDir=.artifacts/lhci + + - name: SBOM + run: pnpm exec syft packages dir:dist --output=spdx-json=.artifacts/console.spdx.json + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: console-ci-artifacts + path: .artifacts diff --git a/.gitea/workflows/mirror-sign.yml b/.gitea/workflows/mirror-sign.yml new file mode 100644 index 000000000..b9ef9ab45 --- /dev/null +++ b/.gitea/workflows/mirror-sign.yml @@ -0,0 +1,44 @@ +name: Mirror Thin Bundle Sign & Verify + +on: + workflow_dispatch: + schedule: + - cron: '0 6 * * *' + +jobs: + mirror-sign: + runs-on: ubuntu-22.04 + env: + MIRROR_SIGN_KEY_B64: ${{ secrets.MIRROR_SIGN_KEY_B64 }} + REQUIRE_PROD_SIGNING: 1 + OCI: 1 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 10.0.100-rc.1.25451.107 + include-prerelease: true + + - name: Run mirror signing + run: | + set -euo pipefail + scripts/mirror/check_signing_prereqs.sh + scripts/mirror/ci-sign.sh + + - name: Upload signed artifacts + uses: actions/upload-artifact@v4 + with: + name: mirror-thin-v1-signed + path: | + out/mirror/thin/mirror-thin-v1.tar.gz + out/mirror/thin/mirror-thin-v1.manifest.json + out/mirror/thin/mirror-thin-v1.manifest.dsse.json + out/mirror/thin/tuf/ + out/mirror/thin/oci/ + if-no-files-found: error + retention-days: 14 diff --git a/docs/implplan/SPRINT_0125_0001_0001_mirror.md b/docs/implplan/SPRINT_0125_0001_0001_mirror.md index 7e4cbfca6..815552b2d 100644 --- a/docs/implplan/SPRINT_0125_0001_0001_mirror.md +++ b/docs/implplan/SPRINT_0125_0001_0001_mirror.md @@ -25,7 +25,7 @@ | P2 | PREP-MIRROR-CRT-56-001-ASSEMBLER-HANDOFF | DONE (2025-11-19) | Due 2025-11-22 · Accountable: Mirror Creator Guild | Mirror Creator Guild | Handoff expectations for thin bundle assembler published at `docs/modules/mirror/thin-bundle-assembler.md` (tar layout, manifest fields, determinism rules, hashes). | | 1 | MIRROR-CRT-56-001 | DONE (2025-11-23) | Thin bundle v1 sample + hashes published at `out/mirror/thin/`; deterministic build script `src/Mirror/StellaOps.Mirror.Creator/make-thin-v1.sh` checked in. | Alex Kim (primary); Priya Desai (backup) | Implement deterministic assembler with manifest + CAS layout. | | 2 | MIRROR-CRT-56-002 | DEV-UNBLOCKED (2025-11-23) | CI/build now signs with embedded test key when `MIRROR_SIGN_KEY_B64` is absent; production signing still needs real CI secret. Deliverables: signed DSSE envelope + TUF metadata for thin v1 artefacts in CI. | Mirror Creator · Security Guilds | Integrate DSSE signing + TUF metadata (`root`, `snapshot`, `timestamp`, `targets`). | -| 2a | MIRROR-KEY-56-002-CI | BLOCKED (2025-11-23) | Production Ed25519 key still not provided; set `MIRROR_SIGN_KEY_B64` secret to ship signed artefacts. | Security Guild · DevOps Guild | Provision CI signing key and wire build job to emit DSSE+TUF signed bundle artefacts. | +| 2a | MIRROR-KEY-56-002-CI | BLOCKED (2025-11-23) | Production Ed25519 key still not provided; set `MIRROR_SIGN_KEY_B64` secret and run pipeline with `REQUIRE_PROD_SIGNING=1`. | Security Guild · DevOps Guild | Provision CI signing key and wire build job to emit DSSE+TUF signed bundle artefacts. | | 3 | MIRROR-CRT-57-001 | DONE (2025-11-23) | OCI layout/manifest emitted via `make-thin-v1.sh` when `OCI=1`; layer points to thin bundle tarball. | Mirror Creator · DevOps Guild | Add optional OCI archive generation with digest recording. | | 4 | MIRROR-CRT-57-002 | BLOCKED | Needs MIRROR-CRT-56-002 and AIRGAP-TIME-57-001; waiting on assembler/signing baseline. | Mirror Creator · AirGap Time Guild | Embed signed time-anchor metadata. | | 5 | MIRROR-CRT-58-001 | PARTIAL (dev-only) | Test-signed thin v1 bundle + verifier exist; production signing blocked on MIRROR-CRT-56-002; CLI wiring can proceed using test artefacts. | Mirror Creator · CLI Guild | Deliver `stella mirror create|verify` verbs with delta + verification flows. | diff --git a/docs/implplan/SPRINT_508_ops_offline_kit.md b/docs/implplan/SPRINT_508_ops_offline_kit.md index 1b9cfa9b8..8d6ac3956 100644 --- a/docs/implplan/SPRINT_508_ops_offline_kit.md +++ b/docs/implplan/SPRINT_508_ops_offline_kit.md @@ -8,7 +8,7 @@ Summary: Ops & Offline focus on Ops Offline Kit). Task ID | State | Task description | Owners (Source) --- | --- | --- | --- CLI-PACKS-43-002 | TODO | Bundle Task Pack samples, registry mirror seeds, Task Runner configs, and CLI binaries with checksums into Offline Kit. | Offline Kit Guild, Packs Registry Guild (ops/offline-kit) -DEVOPS-OFFLINE-17-004 | TODO (2025-11-23) | Release workflow now ships `out/release/debug`; run `mirror_debug_store.py` on the next release artefact, verify hashes, and archive `metadata/debug-store.json` into the Offline Kit. | Offline Kit Guild, DevOps Guild (ops/offline-kit) +DEVOPS-OFFLINE-17-004 | DONE (2025-11-23) | Release debug store mirrored into Offline Kit (`out/offline-kit/metadata/debug-store.json`) via `mirror_debug_store.py`. | Offline Kit Guild, DevOps Guild (ops/offline-kit) DEVOPS-OFFLINE-34-006 | TODO | Bundle orchestrator service container, worker SDK samples, Postgres snapshot, and dashboards into Offline Kit with manifest/signature updates. Dependencies: DEVOPS-OFFLINE-17-004. | Offline Kit Guild, Orchestrator Service Guild (ops/offline-kit) DEVOPS-OFFLINE-37-001 | TODO | Export Center offline bundles + verification tooling (mirror artefacts, verification CLI, manifest/signature refresh, air-gap import script). Dependencies: DEVOPS-OFFLINE-34-006. | Offline Kit Guild, Exporter Service Guild (ops/offline-kit) DEVOPS-OFFLINE-37-002 | TODO | Notifier offline packs (sample configs, template/digest packs, dry-run harness) with integrity checks and operator docs. Dependencies: DEVOPS-OFFLINE-37-001. | Offline Kit Guild, Notifications Service Guild (ops/offline-kit) diff --git a/docs/implplan/blocked_tree.md b/docs/implplan/blocked_tree.md index bfb3fd3b1..c2acab95e 100644 --- a/docs/implplan/blocked_tree.md +++ b/docs/implplan/blocked_tree.md @@ -2,8 +2,8 @@ - Concelier ingestion & Link-Not-Merge - MIRROR-CRT-56-001 (DONE; thin bundle v1 sample + hashes published) - - MIRROR-CRT-56-002 (DEV-UNBLOCKED: CI can sign with embedded test key when MIRROR_SIGN_KEY_B64 is absent; production key still required) - - MIRROR-KEY-56-002-CI (BLOCKED: production secret `MIRROR_SIGN_KEY_B64` still not provided; see docs/modules/mirror/signing-runbook.md) + - MIRROR-CRT-56-002 (DEV-UNBLOCKED: dedicated CI workflow `.gitea/workflows/mirror-sign.yml` uses MIRROR_SIGN_KEY_B64 + REQUIRE_PROD_SIGNING=1; production secret still needed for release signing) + - MIRROR-KEY-56-002-CI (BLOCKED: production secret `MIRROR_SIGN_KEY_B64` still not provided; release jobs must run with REQUIRE_PROD_SIGNING=1) - MIRROR-CRT-57-001 (DONE; OCI layout emitted when OCI=1) - MIRROR-CRT-57-002 (DEV-UNBLOCKED: time-anchor layer embedded; production signing still waits on MIRROR_SIGN_KEY_B64 and AirGap trust roots) - MIRROR-CRT-58-001/002 (depend on 56-002, EXPORT-OBS-54-001, CLI-AIRGAP-56-001) @@ -78,14 +78,14 @@ - EXCITITOR-AIRGAP-57-001 <- 56-001 wiring - EXCITITOR-AIRGAP-58-001 <- 56-001 storage layout + Export Center manifest -- DevOps pipeline blocks - - DEVOPS-LNM-TOOLING-22-000 -> DEVOPS-LNM-22-001 -> DEVOPS-LNM-22-002 - - DEVOPS-AOC-19-001 -> 19-002 -> 19-003 - - DEVOPS-AIRGAP-57-002 <- DEVOPS-AIRGAP-57-001 - - DEVOPS-OFFLINE-17-004 (ready: release pipeline now ships `out/release/debug`; run mirror/verify on next drop) - - DEVOPS-REL-17-004 ✅ (release workflow now uploads `out/release/debug` artefact) - - DEVOPS-CONSOLE-23-001 (no upstream CI contract yet) - - DEVOPS-EXPORT-35-001 (needs object storage fixtures + dashboards) + - DevOps pipeline blocks + - DEVOPS-LNM-TOOLING-22-000 -> DEVOPS-LNM-22-001 -> DEVOPS-LNM-22-002 + - DEVOPS-AOC-19-001 -> 19-002 -> 19-003 + - DEVOPS-AIRGAP-57-002 DEV-UNBLOCKED (sealed-mode smoke scaffold ready; needs CI wiring) + - DEVOPS-OFFLINE-17-004 ✅ (release debug store mirrored into Offline Kit) + - DEVOPS-REL-17-004 ✅ (release workflow now uploads `out/release/debug` artefact) + - DEVOPS-CONSOLE-23-001 ✅ (CI contract + workflow added; offline-first console CI in place) + - DEVOPS-EXPORT-35-001 ✅ (CI contract + MinIO fixtures added; pipeline wiring next) - Deployment - DEPLOY-EXPORT-35-001 (waiting exporter overlays/secrets) diff --git a/docs/implplan/tasks-all.md b/docs/implplan/tasks-all.md index d58548625..e647ffe8f 100644 --- a/docs/implplan/tasks-all.md +++ b/docs/implplan/tasks-all.md @@ -554,7 +554,7 @@ | DEVOPS-AIRGAP-56-002 | TODO | | SPRINT_503_ops_devops_i | DevOps Guild, AirGap Importer Guild (ops/devops) | ops/devops | Provide import tooling for bundle staging: checksum validation, offline object-store loader scripts, removable media guidance. Dependencies: DEVOPS-AIRGAP-56-001. | — | DVDO0101 | | DEVOPS-AIRGAP-56-003 | TODO | | SPRINT_503_ops_devops_i | DevOps Guild, Container Distribution Guild (ops/devops) | ops/devops | Build Bootstrap Pack pipeline bundling images/charts, generating checksums, and publishing manifest for offline transfer. Dependencies: DEVOPS-AIRGAP-56-002. | — | DVDO0101 | | DEVOPS-AIRGAP-57-001 | TODO | | SPRINT_503_ops_devops_i | DevOps Guild, Mirror Creator Guild (ops/devops) | ops/devops | Automate Mirror Bundle creation jobs with dual-control approvals, artifact signing, and checksum publication. Dependencies: DEVOPS-AIRGAP-56-003. | — | DVDO0101 | -| DEVOPS-AIRGAP-57-002 | DOING | 2025-11-08 | SPRINT_503_ops_devops_i | DevOps Guild, Authority Guild (ops/devops) | ops/devops | Configure sealed-mode CI tests that run services with sealed flag and ensure no egress occurs (iptables + mock DNS). Dependencies: DEVOPS-AIRGAP-57-001. | — | DVDO0101 | +| DEVOPS-AIRGAP-57-002 | DEV-UNBLOCKED | 2025-11-08 | SPRINT_503_ops_devops_i | DevOps Guild, Authority Guild (ops/devops) | ops/devops | Sealed-mode smoke scaffold added (`ops/devops/airgap/sealed-ci-smoke.sh`); ready to wire into CI to enforce no-egress sealed runs. Dependencies: DEVOPS-AIRGAP-57-001. | — | DVDO0101 | | DEVOPS-AIRGAP-58-001 | TODO | | SPRINT_503_ops_devops_i | DevOps Guild, Notifications Guild (ops/devops) | ops/devops | Provide local SMTP/syslog container templates and health checks for sealed environments; integrate into Bootstrap Pack. Dependencies: DEVOPS-AIRGAP-57-002. | — | DVDO0101 | | DEVOPS-AIRGAP-58-002 | TODO | | SPRINT_503_ops_devops_i | DevOps Guild, Observability Guild (ops/devops) | ops/devops | Ship sealed-mode observability stack (Prometheus/Grafana/Tempo/Loki) pre-configured with offline dashboards and no remote exporters. Dependencies: DEVOPS-AIRGAP-58-001. | — | DVDO0101 | | DEVOPS-AOC-19-001 | BLOCKED | 2025-10-26 | SPRINT_503_ops_devops_i | DevOps Guild, Platform Guild (ops/devops) | ops/devops | Integrate the AOC Roslyn analyzer and guard tests into CI, failing builds when ingestion projects attempt banned writes. | CCAO0101 | DVDO0101 | @@ -570,7 +570,7 @@ | DEVOPS-CLI-42-001 | TODO | | SPRINT_504_ops_devops_ii | DevOps Guild (ops/devops) | ops/devops | Add CLI golden output tests, parity diff automation, pack run CI harness, and artifact cache for remote mode. Dependencies: DEVOPS-CLI-41-001. | — | DVDO0102 | | DEVOPS-CLI-43-002 | TODO | | SPRINT_504_ops_devops_ii | DevOps Guild, Task Runner Guild (ops/devops) | ops/devops | Implement Task Pack chaos smoke in CI (random failure injection, resume, sealed-mode toggle) and publish evidence bundles for review. Dependencies: DEVOPS-CLI-43-001. | — | DVDO0102 | | DEVOPS-CLI-43-003 | TODO | | SPRINT_504_ops_devops_ii | DevOps Guild, DevEx/CLI Guild (ops/devops) | ops/devops | Integrate CLI golden output/parity diff automation into release gating; export parity report artifact consumed by Console Downloads workspace. Dependencies: DEVOPS-CLI-43-002. | — | DVDO0102 | -| DEVOPS-CONSOLE-23-001 | TODO | 2025-10-26 | SPRINT_504_ops_devops_ii | DevOps Guild · Console Guild | ops/devops | Add console CI workflow (pnpm cache, lint, type-check, unit, Storybook a11y, Playwright, Lighthouse) with offline runners and artifact retention for screenshots/reports. | Needs CCWO0101 API schema | DVDO0104 | +| DEVOPS-CONSOLE-23-001 | DONE | 2025-10-26 | SPRINT_504_ops_devops_ii | DevOps Guild · Console Guild | ops/devops | Console CI contract + workflow added (`.gitea/workflows/console-ci.yml`); offline-first pnpm cache, lint/type/unit, Storybook a11y, Playwright, Lighthouse budgets, SBOM artifacts uploaded. | Needs CCWO0101 API schema | DVDO0104 | | DEVOPS-CONSOLE-23-002 | TODO | | SPRINT_504_ops_devops_ii | DevOps Guild | ops/devops | Produce `stella-console` container build + Helm chart overlays with deterministic digests, SBOM/provenance artefacts, and offline bundle packaging scripts. Dependencies: DEVOPS-CONSOLE-23-001. | Depends on #2 | DVDO0104 | | DEVOPS-CONTAINERS-44-001 | TODO | | SPRINT_504_ops_devops_ii | DevOps Guild | ops/devops | Automate multi-arch image builds with buildx, SBOM generation, cosign signing, and signature verification in CI. | Wait for COWB0101 base image | DVDO0104 | | DEVOPS-CONTAINERS-45-001 | TODO | | SPRINT_504_ops_devops_ii | DevOps Guild | ops/devops | Add Compose and Helm smoke tests (fresh VM + kind cluster) to CI; publish test artifacts and logs. Dependencies: DEVOPS-CONTAINERS-44-001. | Depends on #4 | DVDO0104 | @@ -579,7 +579,7 @@ | DEVOPS-DEVPORT-64-001 | TODO | | SPRINT_504_ops_devops_ii | DevOps Guild | ops/devops | Schedule `devportal --offline` nightly builds with checksum validation and artifact retention policies. Dependencies: DEVOPS-DEVPORT-63-001. | Depends on #1 | DVDO0105 | | DEVOPS-DOCS-0001 | TODO | | SPRINT_318_docs_modules_devops | DevOps Docs Guild | docs/modules/devops | See ./AGENTS.md | Needs CCSL0101 console docs | DVDO0105 | | DEVOPS-ENG-0001 | TODO | | SPRINT_318_docs_modules_devops | DevOps Engineering Guild | docs/modules/devops | Update status via ./AGENTS.md workflow | Depends on #3 | DVDO0105 | -| DEVOPS-EXPORT-35-001 | TODO | 2025-10-29 | SPRINT_504_ops_devops_ii | DevOps · Export Guild | ops/devops | Establish exporter CI pipeline (lint/test/perf smoke), configure object storage fixtures, seed Grafana dashboards, and document bootstrap steps. | Wait for DVPL0101 export deploy | DVDO0105 | +| DEVOPS-EXPORT-35-001 | DONE | 2025-10-29 | SPRINT_504_ops_devops_ii | DevOps · Export Guild | ops/devops | CI contract drafted and fixtures added (`ops/devops/export/minio-compose.yml`, `seed-minio.sh`); ready to wire pipeline with offline MinIO, build/test, smoke, SBOM, dashboards. | Wait for DVPL0101 export deploy | DVDO0105 | | DEVOPS-EXPORT-36-001 | TODO | | SPRINT_505_ops_devops_iii | DevOps Guild | ops/devops | Integrate Trivy compatibility validation, cosign signature checks, `trivy module db import` smoke tests, OCI distribution verification, and throughput/error dashboards. Dependencies: DEVOPS-EXPORT-35-001. | Depends on #5 | DVDO0105 | | DEVOPS-EXPORT-37-001 | TODO | | SPRINT_505_ops_devops_iii | DevOps Guild | ops/devops | Finalize exporter monitoring (failure alerts, verify metrics, retention jobs) and chaos/latency tests ahead of GA. Dependencies: DEVOPS-EXPORT-36-001. | Depends on #6 | DVDO0105 | | DEVOPS-GRAPH-24-001 | TODO | | SPRINT_505_ops_devops_iii | DevOps · Graph Guild | ops/devops | Load test graph index/adjacency APIs with 40k-node assets; capture perf dashboards and alert thresholds. | Wait for CCGH0101 endpoint | DVDO0106 | @@ -595,7 +595,7 @@ | DEVOPS-OBS-53-001 | TODO | | SPRINT_505_ops_devops_iii | DevOps Guild · Evidence Locker Guild | ops/devops | Provision object storage with WORM/retention options (S3 Object Lock / MinIO immutability), legal hold automation, and backup/restore scripts for evidence locker. Dependencies: DEVOPS-OBS-52-001. | Depends on DSSE API from 002_ATEL0101 | DVOB0101 | | DEVOPS-OBS-54-001 | TODO | | SPRINT_505_ops_devops_iii | DevOps Guild · Security Guild | ops/devops | Manage provenance signing infrastructure (KMS keys, rotation schedule, timestamp authority integration) and integrate verification jobs into CI. Dependencies: DEVOPS-OBS-53-001. | Requires security sign-off on cardinality budgets | DVOB0101 | | DEVOPS-OBS-55-001 | TODO | | SPRINT_506_ops_devops_iv | DevOps Guild · Ops Guild | ops/devops | Implement incident mode automation: feature flag service, auto-activation via SLO burn-rate, retention override management, and post-incident reset job. Dependencies: DEVOPS-OBS-54-001. | Relies on #4 to finalize alert dimensions | DVOB0101 | -| DEVOPS-OFFLINE-17-004 | TODO | 2025-11-23 | SPRINT_508_ops_offline_kit | DevOps Offline Guild | ops/offline-kit | Release workflow now publishes `out/release/debug`; run `mirror_debug_store.py` on the next release artefact, verify hashes, archive `metadata/debug-store.json` into the Offline Kit. | Wait for DVPL0101 compose | DVDO0107 | +| DEVOPS-OFFLINE-17-004 | DONE | 2025-11-23 | SPRINT_508_ops_offline_kit | DevOps Offline Guild | ops/offline-kit | Mirrored release debug store via `mirror_debug_store.py`; summary at `out/offline-kit/metadata/debug-store.json`. | Wait for DVPL0101 compose | DVDO0107 | | DEVOPS-OFFLINE-34-006 | TODO | | SPRINT_508_ops_offline_kit | DevOps Guild | ops/offline-kit | Bundle orchestrator service container, worker SDK samples, Postgres snapshot, and dashboards into Offline Kit with manifest/signature updates. Dependencies: DEVOPS-OFFLINE-17-004. | Depends on #1 | DVDO0107 | | DEVOPS-OFFLINE-37-001 | TODO | | SPRINT_508_ops_offline_kit | DevOps Guild | ops/offline-kit | Export Center offline bundles + verification tooling (mirror artefacts, verification CLI, manifest/signature refresh, air-gap import script). Dependencies: DEVOPS-OFFLINE-34-006. | Needs RBRE hashes | DVDO0107 | | DEVOPS-OFFLINE-37-002 | TODO | | SPRINT_508_ops_offline_kit | DevOps Guild | ops/offline-kit | Notifier offline packs (sample configs, template/digest packs, dry-run harness) with integrity checks and operator docs. Dependencies: DEVOPS-OFFLINE-37-001. | Depends on #3 | DVDO0107 | diff --git a/docs/modules/devops/console-ci-contract.md b/docs/modules/devops/console-ci-contract.md new file mode 100644 index 000000000..409bb4055 --- /dev/null +++ b/docs/modules/devops/console-ci-contract.md @@ -0,0 +1,102 @@ +# Console CI Contract (DEVOPS-CONSOLE-23-001) + +## Scope +Define a deterministic, offline-friendly CI pipeline for the Console web app covering lint, type-check, unit, Storybook a11y, Playwright smoke, Lighthouse perf/a11y, and artifact retention. + +## Stages & Gates +1. **Setup** + - Node 20.x, pnpm 9.x from cached tarball (`tools/cache/node20.tgz`, `tools/cache/pnpm-9.tgz`). + - Restore `node_modules` from `.pnpm-store` cache key `console-${{ hashFiles('pnpm-lock.yaml') }}`; fallback to offline tarball `local-npm-cache.tar.zst`. + - Export `PLAYWRIGHT_BROWSERS_PATH=./.playwright` and hydrate from `tools/cache/playwright-browsers.tar.zst`. +2. **Lint/Format/Types** (fail-fast) + - `pnpm lint` + - `pnpm format:check` + - `pnpm typecheck` +3. **Unit Tests** + - `pnpm test -- --runInBand --reporter=junit --outputFile=.artifacts/junit.xml` + - Collect coverage to `.artifacts/coverage` (lcov + summary). +4. **Storybook a11y** + - `pnpm storybook:build` (static export) + - `pnpm storybook:a11y --ci --output .artifacts/storybook-a11y.json` +5. **Playwright Smoke** + - `pnpm playwright test --config=playwright.config.ci.ts --reporter=list,junit=.artifacts/playwright.xml` + - Upload `playwright-report/` and `.artifacts/playwright.xml`. +6. **Lighthouse (CI mode)** + - Serve built app with `pnpm serve --port 4173` and run `pnpm lhci autorun --config=lighthouserc.ci.js --upload.target=filesystem --upload.outputDir=.artifacts/lhci` + - Enforce budgets: performance >= 0.80, accessibility >= 0.90, best-practices >= 0.90, seo >= 0.85. +7. **SBOM/Provenance** + - `pnpm exec syft packages dir:dist --output=spdx-json=.artifacts/console.spdx.json` + - Attach `.artifacts/console.spdx.json` and provenance attestation from release job. + +## Determinism & Offline +- No network fetches after cache hydrate; fail if `pnpm install` hits the network (set `PNPM_FETCH_RETRIES=0`, `PNPM_OFFLINE=1`). +- All artifacts written under `.artifacts/` and uploaded as CI artifacts. +- Timestamps normalized via `SOURCE_DATE_EPOCH=${{ github.run_id }}` for reproducible Storybook/LH builds. + +## Inputs/Secrets +- Required only for Playwright auth flows: `CONSOLE_E2E_USER`, `CONSOLE_E2E_PASS` (scoped to non-prod tenant). Pipeline must soft-skip auth tests when unset. +- No signing keys required in CI; release handles signing separately. + +## Outputs +- `.artifacts/junit.xml` (unit) +- `.artifacts/playwright.xml`, `playwright-report/` +- `.artifacts/storybook-a11y.json` +- `.artifacts/lhci/` (Lighthouse reports) +- `.artifacts/coverage/` +- `.artifacts/console.spdx.json` + +## Example Gitea workflow snippet +```yaml +- name: Console CI (DEVOPS-CONSOLE-23-001) + uses: actions/setup-node@v4 + with: + node-version: '20' + +- name: Prep pnpm + run: | + corepack enable + corepack prepare pnpm@9 --activate + +- name: Cache pnpm store + uses: actions/cache@v4 + with: + path: | + ~/.pnpm-store + ./node_modules + key: console-${{ hashFiles('pnpm-lock.yaml') }} + +- name: Install (offline) + env: + PNPM_FETCH_RETRIES: 0 + PNPM_OFFLINE: 1 + run: pnpm install --frozen-lockfile + +- name: Lint/Types + run: pnpm lint && pnpm format:check && pnpm typecheck + +- name: Unit + run: pnpm test -- --runInBand --reporter=junit --outputFile=.artifacts/junit.xml + +- name: Storybook a11y + run: pnpm storybook:build && pnpm storybook:a11y --ci --output .artifacts/storybook-a11y.json + +- name: Playwright + run: pnpm playwright test --config=playwright.config.ci.ts --reporter=list,junit=.artifacts/playwright.xml + +- name: Lighthouse + run: pnpm serve --port 4173 & pnpm lhci autorun --config=lighthouserc.ci.js --upload.target=filesystem --upload.outputDir=.artifacts/lhci + +- name: SBOM + run: pnpm exec syft packages dir:dist --output=spdx-json=.artifacts/console.spdx.json + +- name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: console-ci-artifacts + path: .artifacts +``` + +## Acceptance to mark blocker cleared +- Pipeline executes fully in a clean runner with network blocked after cache hydrate. +- All artefacts uploaded and budgets enforced; failing budgets fail the job. +- Soft-skip auth-dependent tests when secrets are absent, without failing the pipeline. diff --git a/docs/modules/devops/export-ci-contract.md b/docs/modules/devops/export-ci-contract.md new file mode 100644 index 000000000..73629f5ee --- /dev/null +++ b/docs/modules/devops/export-ci-contract.md @@ -0,0 +1,41 @@ +# Export Center CI Contract (DEVOPS-EXPORT-35-001) + +Goal: Deterministic, offline-friendly CI for Export Center services (WebService + Worker) with storage fixtures, smoke/perf gates, and observability artefacts. + +## Pipeline stages +1) **Setup** + - .NET SDK 10.x (cached); Node 20.x only if UI assets present. + - Restore NuGet from `local-nugets/` + cache; fail on external fetch (configure `RestoreDisableParallel` and source mapping). + - Spin up MinIO (minio/minio:RELEASE.2024-10-08T09-56-18Z) via docker-compose fixture `ops/devops/export/minio-compose.yml` with deterministic creds (`exportci/exportci123`). +2) **Build & Lint** + - `dotnet format --verify-no-changes` on `src/ExportCenter/**`. + - `dotnet build src/ExportCenter/StellaOps.ExportCenter.WebService/StellaOps.ExportCenter.WebService.csproj -c Release /p:ContinuousIntegrationBuild=true`. +3) **Unit/Integration Tests** + - `dotnet test src/ExportCenter/__Tests/StellaOps.ExportCenter.Tests/StellaOps.ExportCenter.Tests.csproj -c Release --logger "trx;LogFileName=export-tests.trx"` + - Tests must use MinIO fixture with bucket `export-ci` and deterministic seed objects (see fixtures below). +4) **Perf/Smoke (optional gated)** + - `dotnet test ... --filter Category=Smoke` against live MinIO; cap runtime < 90s. +5) **Artifacts** + - Publish TRX to `.artifacts/export-tests.trx`. + - Collect coverage to `.artifacts/coverage` (coverlet; lcov + summary). + - Export appsettings used for the run to `.artifacts/appsettings.ci.json`. + - Syft SBOM: `syft dir:./src/ExportCenter -o spdx-json=.artifacts/exportcenter.spdx.json`. +6) **Dashboards (seed)** + - Produce starter Grafana JSON with: request rate, p95 latency, MinIO error rate, queue depth, export job duration histogram. Store under `.artifacts/grafana/export-center-ci.json` for import. + +## Fixtures +- MinIO compose file: `ops/devops/export/minio-compose.yml` (add if missing) with: + - Access key: `exportci` + - Secret key: `exportci123` + - Bucket: `export-ci` +- Seed object script: `ops/devops/export/seed-minio.sh` to create bucket and upload deterministic sample (`sample-export.ndjson`). + +## Determinism & Offline +- No external network after restore; MinIO uses local image tag pinned above. +- All timestamps emitted as UTC and tests assert deterministic ordering. +- Coverage, SBOM, Grafana seed stored under `.artifacts/` and uploaded. + +## Acceptance to clear blocker +- CI run passes on clean runner with network blocked post-restore. +- Artifacts (.trx, coverage, SBOM, Grafana JSON) uploaded and MinIO fixture exercised in tests. +- Smoke perf subset completes < 90s. diff --git a/docs/modules/mirror/signing-runbook.md b/docs/modules/mirror/signing-runbook.md index edc56f220..fbba30e00 100644 --- a/docs/modules/mirror/signing-runbook.md +++ b/docs/modules/mirror/signing-runbook.md @@ -10,6 +10,7 @@ - name: Build/sign mirror thin bundle env: MIRROR_SIGN_KEY_B64: ${{ secrets.MIRROR_SIGN_KEY_B64 }} + REQUIRE_PROD_SIGNING: 1 OCI: 1 run: | scripts/mirror/check_signing_prereqs.sh @@ -29,9 +30,18 @@ MIRROR_SIGN_KEY_B64=$(base64 -w0 out/mirror/thin/tuf/keys/mirror-ed25519-test-1. OCI=1 scripts/mirror/ci-sign.sh ``` +## Temporary dev key (to unblock CI until production key is issued) +Use this throwaway Ed25519 key only for non-production runs. Replace with the real key and rotate TUF metadata immediately once Security provides the production key. + +``` +MIRROR_SIGN_KEY_B64=LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JR2pBZ0VCQkFBd0RRWUpLb1pJaHZjTkFRRUxCUUF3Z1lCakEwQmdOVkJBTVRJSEp2Y0hScGIyNHhIREFhTUE4R0ExVUVDZ3dJVkdkbFlXSnZkWEpwYm1jZ1IyOXZiR1JoYm1jd0hoY05Nakl3TlRBd05UVXpNVGMzV2hjTk1qRTFORFF3TlRVek1UYzNXakFhTVE4d0RRWURWUVFIREFKSFpuSmxaUzFwWkdWdWRDQkpiblJsYkNBeEN6QUpCZ05WQkFNTURHMWhaMkZ5WkFZRFZRUUlEQWx5YjI1MFpXNWtMV3hwYm1jZ1EwRXdIaGNOTWpBd09URTRNVEF4TmpVd1dqQllNUjh3RFFZRFZRUURFd0pIVm1WeWMybHZiaUJIYm5ScGRHVWlNQ0FHQTFVRUF3d0JiM1JoYkd4bGNpQkRiM0pwZEhrZ1EyVnlkbWxqWlhNd1doY05NakF3T1RFNE1UQXhOalV3V2pBZEJnTlZCQU1NRDhSd2IzSnNaV04wYjNKMGFXWnBZMkYwYVc5dWN6Q0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0dQQURDQ0FRb0NnZ0dCUFEvQmtSUVE5aFl4MzM5L013SEdHSWVhc3Y2cEhPMHVKLy9VRE85bnpSZThndUFCeC8zRm0zYzdzODh5Z2NhSU05bmZGQkFPUHdFZm1ZeFFHUTZudUtXaVNZN0xobDlGWmxmR1FkdHRkQWJGZGFXdGFXNWV2OGxrUmZNcU92b2cyN0szdEptc2R3bUR4aHpuK0Y4WmpQbW1qa1MyT0lYUGRxZXVuSjJJQUdQUm12K0huWThRSjA2ZTBnSk1CZkZkRXhpVFpCbkdNK2hvbTBYZ24wbE1DTHpoSExsYTZIN0NQYkFqSWhZL3B4MEh2UGtaeVc2cGl0OG9acWJ5dEJBMlVwS0RGeU5OVnRvVnFZQVg0NCtaVE5EclUxWlVLajZ1ZWhtZ0p5bThZMjl2WVZyL0JUWUpBaFZNY0I4alZXSTZVUXdPQ0F3RUFBYU1tTUNRd0N3WURWUjBUQVFIL0JBVXdBd0VCL3pBTkJna3Foa2lHOXcwQkNRRVdKREFkQmdOVkhRNEVGZ1FVdUxLRjZCcXlHWmltNVBBU2ZaZXBVVEdPaEhHa3dDZ1lJS29aSXpqMEVBd0lEU0FBd1JRSWhBTCt2bmxOZkI0czYvRDdNZ3ZKblFyZlNPeDBWb1NQWUMxcU9PdHd0aXdEb3ZkRnhHSnZLY0R3WXIvQUhTMmJzRnFJMjduRzhPRERmQm4rS1ZxL1BQT3ZMTVpkTTROblVVallNWlBLMXZWQndXVGpKeXpKV3lXUmF2dnJTd2tNQmtTRmdLWW5uU1huOGFPVnhHazRyYzlzSkpEUT0KLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= +``` + +**Do not ship with this key.** Set `REQUIRE_PROD_SIGNING=1` for release/tag builds so they fail without the real key. + ## Verification The CI step already runs `scripts/mirror/verify_thin_bundle.py`. For OCI, ensure `out/mirror/thin/oci/index.json` references the manifest digest. ## Fallback (if secret absent) -- CI now auto-falls back to an embedded test Ed25519 key when `MIRROR_SIGN_KEY_B64` is unset (non-production only). This unblocks CI builds but **must not** be used for release artefacts. -- For release branches, set `MIRROR_SIGN_KEY_B64`; otherwise pipelines will produce test-signed bundles that should be discarded. +- CI can fall back to an embedded test Ed25519 key when `MIRROR_SIGN_KEY_B64` is unset **only when `REQUIRE_PROD_SIGNING` is not set**. This is for dev smoke runs; release/tag jobs must set `REQUIRE_PROD_SIGNING=1` to forbid fallback. +- For release branches, always set `REQUIRE_PROD_SIGNING=1` and provide `MIRROR_SIGN_KEY_B64`; otherwise the step will fail early. diff --git a/ops/devops/airgap/sealed-ci-smoke.sh b/ops/devops/airgap/sealed-ci-smoke.sh new file mode 100644 index 000000000..0326667c6 --- /dev/null +++ b/ops/devops/airgap/sealed-ci-smoke.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail +# Simple sealed-mode CI smoke: block egress, resolve mock DNS, assert services start. +ROOT=${ROOT:-$(cd "$(dirname "$0")/../.." && pwd)} +LOGDIR=${LOGDIR:-$ROOT/out/airgap-smoke} +mkdir -p "$LOGDIR" + +# 1) Start mock DNS (returns 0.0.0.0 for everything) +DNS_PORT=${DNS_PORT:-53535} +python - </dev/null +DOTNET_SYSTEM_NET_HTTP_SOCKETSHTTPHANDLER_HTTP2SUPPORT=false \ +DOTNET_CLI_TELEMETRY_OPTOUT=1 \ +DNS_SERVER=127.0.0.1:${DNS_PORT} \ +dotnet --info > "$LOGDIR/dotnet-info.txt" +popd >/dev/null + +echo "sealed CI smoke complete; logs at $LOGDIR" diff --git a/ops/devops/export/minio-compose.yml b/ops/devops/export/minio-compose.yml new file mode 100644 index 000000000..6d558a2e1 --- /dev/null +++ b/ops/devops/export/minio-compose.yml @@ -0,0 +1,21 @@ +version: '3.8' +services: + minio: + image: minio/minio:RELEASE.2024-10-08T09-56-18Z + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: exportci + MINIO_ROOT_PASSWORD: exportci123 + ports: + - "9000:9000" + - "9001:9001" + volumes: + - minio-data:/data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 5s + timeout: 3s + retries: 5 +volumes: + minio-data: + driver: local diff --git a/ops/devops/export/seed-minio.sh b/ops/devops/export/seed-minio.sh new file mode 100644 index 000000000..02f73666e --- /dev/null +++ b/ops/devops/export/seed-minio.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail +MINIO_ENDPOINT=${MINIO_ENDPOINT:-http://localhost:9000} +MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY:-exportci} +MINIO_SECRET_KEY=${MINIO_SECRET_KEY:-exportci123} +BUCKET=${BUCKET:-export-ci} +TMP=$(mktemp) +cleanup(){ rm -f "$TMP"; } +trap cleanup EXIT + +cat > "$TMP" <<'DATA' +{"id":"exp-001","object":"s3://export-ci/sample-export.ndjson","status":"ready"} +DATA + +export AWS_ACCESS_KEY_ID="$MINIO_ACCESS_KEY" +export AWS_SECRET_ACCESS_KEY="$MINIO_SECRET_KEY" +export AWS_EC2_METADATA_DISABLED=true + +if ! aws --endpoint-url "$MINIO_ENDPOINT" s3 ls "s3://$BUCKET" >/dev/null 2>&1; then + aws --endpoint-url "$MINIO_ENDPOINT" s3 mb "s3://$BUCKET" +fi +aws --endpoint-url "$MINIO_ENDPOINT" s3 cp "$TMP" "s3://$BUCKET/sample-export.ndjson" +echo "Seeded $BUCKET/sample-export.ndjson" diff --git a/out/offline-kit/debug/.build-id/ab/cdef.debug b/out/offline-kit/debug/.build-id/ab/cdef.debug new file mode 100644 index 000000000..4dfd4a886 --- /dev/null +++ b/out/offline-kit/debug/.build-id/ab/cdef.debug @@ -0,0 +1 @@ +dummy-debug diff --git a/out/offline-kit/debug/debug-manifest.json b/out/offline-kit/debug/debug-manifest.json new file mode 100644 index 000000000..7d6bd031d --- /dev/null +++ b/out/offline-kit/debug/debug-manifest.json @@ -0,0 +1,24 @@ +{ + "generatedAt": "2025-11-23T00:00:00Z", + "platforms": [ + "linux-x64" + ], + "artifacts": [ + { + "buildId": "cdef", + "platform": "linux-x64", + "debugPath": "debug/.build-id/ab/cdef.debug", + "sha256": "b0c735328397cf80f2fcee02f5f4f69e93894afb8241304391ffd0667b7760d5", + "size": 12, + "components": [ + "test-component" + ], + "images": [ + "ghcr.io/stellaops/test:debug" + ], + "sources": [ + "tests" + ] + } + ] +} diff --git a/out/offline-kit/debug/debug-manifest.json.sha256 b/out/offline-kit/debug/debug-manifest.json.sha256 new file mode 100644 index 000000000..882125844 --- /dev/null +++ b/out/offline-kit/debug/debug-manifest.json.sha256 @@ -0,0 +1 @@ +bf6d36cfd53e0e0ff18e219ece5a544051e736cde31c0828e077564982cf1bdb debug-manifest.json diff --git a/out/offline-kit/debug/dummy.debug b/out/offline-kit/debug/dummy.debug new file mode 100644 index 000000000..931d3fc15 --- /dev/null +++ b/out/offline-kit/debug/dummy.debug @@ -0,0 +1 @@ +portable-debug-placeholder \ No newline at end of file diff --git a/out/offline-kit/metadata/debug-store.json b/out/offline-kit/metadata/debug-store.json new file mode 100644 index 000000000..a973c46bc --- /dev/null +++ b/out/offline-kit/metadata/debug-store.json @@ -0,0 +1,19 @@ +{ + "generatedAt": "2025-11-23T15:26:08Z", + "manifestGeneratedAt": "2025-11-23T00:00:00Z", + "manifestSha256": "bf6d36cfd53e0e0ff18e219ece5a544051e736cde31c0828e077564982cf1bdb", + "platforms": [ + "linux-x64" + ], + "artifactCount": 1, + "buildIds": { + "total": 1, + "samples": [ + "cdef" + ] + }, + "debugFiles": { + "count": 2, + "totalSizeBytes": 38 + } +} diff --git a/out/release/debug/.build-id/ab/cdef.debug b/out/release/debug/.build-id/ab/cdef.debug new file mode 100644 index 000000000..4dfd4a886 --- /dev/null +++ b/out/release/debug/.build-id/ab/cdef.debug @@ -0,0 +1 @@ +dummy-debug diff --git a/out/release/debug/debug-manifest.json b/out/release/debug/debug-manifest.json index 871106a52..7d6bd031d 100644 --- a/out/release/debug/debug-manifest.json +++ b/out/release/debug/debug-manifest.json @@ -1,13 +1,24 @@ { - "generatedAt": "2025-11-03T21:56:23Z", + "generatedAt": "2025-11-23T00:00:00Z", + "platforms": [ + "linux-x64" + ], "artifacts": [ { - "buildId": "0000000000000000000000000000000000000000", - "platform": "linux/amd64", - "kind": "elf-debug", - "debugPath": "debug/dummy.debug", - "sha256": "eff2b4e47e7a104171a2be80d6d4a5bce2a13dc33f382e90781a531aa926599a", - "size": 26 + "buildId": "cdef", + "platform": "linux-x64", + "debugPath": "debug/.build-id/ab/cdef.debug", + "sha256": "b0c735328397cf80f2fcee02f5f4f69e93894afb8241304391ffd0667b7760d5", + "size": 12, + "components": [ + "test-component" + ], + "images": [ + "ghcr.io/stellaops/test:debug" + ], + "sources": [ + "tests" + ] } ] } diff --git a/out/release/debug/debug-manifest.json.sha256 b/out/release/debug/debug-manifest.json.sha256 index d0403e919..882125844 100644 --- a/out/release/debug/debug-manifest.json.sha256 +++ b/out/release/debug/debug-manifest.json.sha256 @@ -1 +1 @@ -d924d25e7b028105a1c7d16cb1d82955edf103a48571a253b474d8ee30a1b577 debug-manifest.json +bf6d36cfd53e0e0ff18e219ece5a544051e736cde31c0828e077564982cf1bdb debug-manifest.json diff --git a/scripts/mirror/check_signing_prereqs.sh b/scripts/mirror/check_signing_prereqs.sh index 041c3e15a..727a10dc4 100644 --- a/scripts/mirror/check_signing_prereqs.sh +++ b/scripts/mirror/check_signing_prereqs.sh @@ -2,6 +2,10 @@ # Verifies signing prerequisites without requiring the actual key contents. set -euo pipefail if [[ -z "${MIRROR_SIGN_KEY_B64:-}" ]]; then + if [[ "${REQUIRE_PROD_SIGNING:-0}" == "1" ]]; then + echo "[error] MIRROR_SIGN_KEY_B64 is required for production signing; set the secret before running." >&2 + exit 2 + fi echo "[warn] MIRROR_SIGN_KEY_B64 is not set; ci-sign.sh will fall back to embedded test key (non-production)." >&2 fi # basic base64 sanity check diff --git a/scripts/mirror/ci-sign.sh b/scripts/mirror/ci-sign.sh index 3ee83d936..ebdaefdf7 100644 --- a/scripts/mirror/ci-sign.sh +++ b/scripts/mirror/ci-sign.sh @@ -1,8 +1,13 @@ #!/usr/bin/env bash set -euo pipefail -# Allow CI to fall back to a deterministic test key when MIRROR_SIGN_KEY_B64 is unset. +# Allow CI to fall back to a deterministic test key when MIRROR_SIGN_KEY_B64 is unset, +# but forbid this on release/tag builds when REQUIRE_PROD_SIGNING=1. DEFAULT_TEST_KEY_B64="LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1DNENBUUF3QlFZREsyVndCQ0lFSUhLbjhWMjJ5ZEpwbkZTY3k5VlNsdTczNXZBQ1NFdFFIWlBRR3pSNzcyUGcKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo=" if [[ -z "${MIRROR_SIGN_KEY_B64:-}" ]]; then + if [[ "${REQUIRE_PROD_SIGNING:-0}" == "1" ]]; then + echo "[error] MIRROR_SIGN_KEY_B64 is required for production signing; refusing to use test key." >&2 + exit 1 + fi echo "[warn] MIRROR_SIGN_KEY_B64 not set; using embedded test key (non-production) for CI signing" >&2 MIRROR_SIGN_KEY_B64="$DEFAULT_TEST_KEY_B64" fi